mirrors
/
SDWebImage
mirror of https://github.com/SDWebImage/SDWebImage.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Replace NSFileManager.enumeratorAtPath with enumeratorAtURL for performance and RAM saving (#3690) 

* fix #3689

1. repalce @selector(enumeratorAtURL:) with @selector(enumeratorAtURL:)
2. replace ioQueueAttributes from DISPATCH_QUEUE_SERIAL to DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL >= iOS 10

* fix: remove NSDirectoryEnumerationProducesRelativePathURLs option

* feat: replace enumeratorAtPath:

* fix: update test44DiskCacheMigrationFromOldVersion

---------

Co-authored-by: huangchengzhi <huangchengzhi@bytedance.com>
This commit is contained in:
ChengzhiHuang 2024-03-26 17:15:42 +08:00 committed by GitHub
parent b156318507
commit d5e3e7f7c5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 133 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
SDWebImage/Core/SDDiskCache.m
View File

@ -165,7 +165,7 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
NSArray<NSString *> *resourceKeys = @[NSURLIsDirectoryKey, cacheContentDateKey, NSURLTotalFileAllocatedSizeKey];
// This enumerator prefetches useful properties for our cache files.
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
includingPropertiesForKeys:resourceKeys
options:NSDirectoryEnumerationSkipsHiddenFiles
errorHandler:NULL];
@ -180,25 +180,27 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
// 2. Storing file attributes for the size-based cleanup pass.
NSMutableArray<NSURL *> *urlsToDelete = [[NSMutableArray alloc] init];
for (NSURL *fileURL in fileEnumerator) {
NSError *error;
NSDictionary<NSString *, id> *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error];
// Skip directories and errors.
if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) {
continue;
@autoreleasepool {
NSError *error;
NSDictionary<NSString *, id> *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error];
// Skip directories and errors.
if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) {
continue;
}
// Remove files that are older than the expiration date;
NSDate *modifiedDate = resourceValues[cacheContentDateKey];
if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
[urlsToDelete addObject:fileURL];
continue;
}
// Store a reference to this file and account for its total size.
NSNumber *totalAllocatedSize = resourceValues[NSURLTotalFileAllocatedSizeKey];
currentCacheSize += totalAllocatedSize.unsignedIntegerValue;
cacheFiles[fileURL] = resourceValues;
}
// Remove files that are older than the expiration date;
NSDate *modifiedDate = resourceValues[cacheContentDateKey];
if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
[urlsToDelete addObject:fileURL];
continue;
}
// Store a reference to this file and account for its total size.
NSNumber *totalAllocatedSize = resourceValues[NSURLTotalFileAllocatedSizeKey];
currentCacheSize += totalAllocatedSize.unsignedIntegerValue;
cacheFiles[fileURL] = resourceValues;
}
for (NSURL *fileURL in urlsToDelete) {
@ -240,19 +242,37 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
- (NSUInteger)totalSize {
NSUInteger size = 0;
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath];
for (NSString *fileName in fileEnumerator) {
NSString *filePath = [self.diskCachePath stringByAppendingPathComponent:fileName];
NSDictionary<NSString *, id> *attrs = [self.fileManager attributesOfItemAtPath:filePath error:nil];
size += [attrs fileSize];
// Use URL-based enumerator instead of Path(NSString *)-based enumerator to reduce
// those objects(ex. NSPathStore2/_NSCFString/NSConcreteData) created during traversal.
// Even worse, those objects are added into AutoreleasePool, in background threads,
// the time to release those objects is undifined(according to the usage of CPU)
// It will truely consumes a lot of VM, up to cause OOMs.
@autoreleasepool {
NSURL *pathURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES];
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL
includingPropertiesForKeys:@[NSURLFileSizeKey]
options:(NSDirectoryEnumerationOptions)0
errorHandler:NULL];
for (NSURL *fileURL in fileEnumerator) {
@autoreleasepool {
NSNumber *fileSize;
[fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL];
size += fileSize.unsignedIntegerValue;
}
}
}
return size;
}
- (NSUInteger)totalCount {
NSUInteger count = 0;
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath];
count = fileEnumerator.allObjects.count;
@autoreleasepool {
NSURL *diskCacheURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES];
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL includingPropertiesForKeys:@[] options:(NSDirectoryEnumerationOptions)0 errorHandler:nil];
count = fileEnumerator.allObjects.count;
}
return count;
}
@ -295,13 +315,21 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
}
} else {
// New directory exist, merge the files
NSDirectoryEnumerator *dirEnumerator = [self.fileManager enumeratorAtPath:srcPath];
NSString *file;
while ((file = [dirEnumerator nextObject])) {
[self.fileManager moveItemAtPath:[srcPath stringByAppendingPathComponent:file] toPath:[dstPath stringByAppendingPathComponent:file] error:nil];
NSURL *srcURL = [NSURL fileURLWithPath:srcPath isDirectory:YES];
NSDirectoryEnumerator<NSURL *> *srcDirEnumerator = [self.fileManager enumeratorAtURL:srcURL
includingPropertiesForKeys:@[]
options:(NSDirectoryEnumerationOptions)0
errorHandler:NULL];
for (NSURL *url in srcDirEnumerator) {
@autoreleasepool {
NSString *dstFilePath = [dstPath stringByAppendingPathComponent:url.lastPathComponent];
NSURL *dstFileURL = [NSURL fileURLWithPath:dstFilePath isDirectory:NO];
[self.fileManager moveItemAtURL:url toURL:dstFileURL error:nil];
}
}
// Remove the old path
[self.fileManager removeItemAtPath:srcPath error:nil];
[self.fileManager removeItemAtURL:srcURL error:nil];
}
}

2
SDWebImage/Core/SDImageCacheConfig.h
View File

@ -129,7 +129,7 @@ typedef NS_ENUM(NSUInteger, SDImageCacheConfigExpireType) {
/**
* The dispatch queue attr for ioQueue. You can config the QoS and concurrent/serial to internal IO queue. The ioQueue is used by SDImageCache to access read/write for disk data.
* Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL), to use serial dispatch queue to ensure single access for disk data. It's safe but may be slow.
* Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL) under iOS 10, `DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL` above and equal iOS 10, using serial dispatch queue is to ensure single access for disk data. It's safe but may be slow.
* @note You can override this to use `DISPATCH_QUEUE_CONCURRENT`, use concurrent queue.
* @warning **MAKE SURE** to keep `diskCacheWritingOptions` to use `NSDataWritingAtomic`, or concurrent queue may cause corrupted disk data (because multiple threads read/write same file without atomic is not IO-safe).
* @note This value does not support dynamic changes. Which means further modification on this value after cache initialized has no effect.

6
SDWebImage/Core/SDImageCacheConfig.m
View File

@ -36,7 +36,11 @@ static const NSInteger kDefaultCacheMaxDiskAge = 60 * 60 * 24 * 7; // 1 week
_maxDiskSize = 0;
_diskCacheExpireType = SDImageCacheConfigExpireTypeModificationDate;
_fileManager = nil;
_ioQueueAttributes = DISPATCH_QUEUE_SERIAL; // NULL
if (@available(iOS 10.0, tvOS 10.0, macOS 10.12, watchOS 3.0, *)) {
_ioQueueAttributes = DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL; // DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM
} else {
_ioQueueAttributes = DISPATCH_QUEUE_SERIAL; // NULL
}
_memoryCacheClass = [SDMemoryCache class];
_diskCacheClass = [SDDiskCache class];
}

23
Tests/Tests/SDImageCacheTests.m
View File

@ -559,18 +559,31 @@ static NSString *kTestImageKeyPNG = @"TestImageKey.png";
NSFileManager *fileManager = [[NSFileManager alloc] init];
config.fileManager = fileManager;
// Fake to store a.png into old path
// Fake to store a%@.png into old path
NSString *newDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"com.hackemist.SDImageCache"] stringByAppendingPathComponent:@"default"];
NSString *oldDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"default"] stringByAppendingPathComponent:@"com.hackemist.SDWebImageCache.default"];
[fileManager createDirectoryAtPath:oldDefaultPath withIntermediateDirectories:YES attributes:nil error:nil];
[fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:@"a.png"] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil];
// Create 100 files to Migrate
for (NSUInteger i = 0; i < 100; i++) {
NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)];
[fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:fileName] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil];
}
// Call migration
SDDiskCache *diskCache = [[SDDiskCache alloc] initWithCachePath:newDefaultPath config:config];
[diskCache moveCacheDirectoryFromPath:oldDefaultPath toPath:newDefaultPath];
// Expect a.png into new path
BOOL exist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:@"a.png"]];
expect(exist).beTruthy();
// Expect a%@.png into new path and oldDefaultPath is deleted
BOOL isDirectory = NO;
for (NSUInteger i = 0; i < 100; i++) {
NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)];
BOOL newFileExist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:fileName] isDirectory:&isDirectory];
expect(newFileExist).beTruthy();
expect(isDirectory).beFalsy();
}
BOOL oldDefaultPathExist = [fileManager fileExistsAtPath:oldDefaultPath];
expect(oldDefaultPathExist).beFalsy();
}
- (void)test45DiskCacheRemoveExpiredData {

62
Tests/Tests/SDWebImageTestCache.m
View File

@ -72,9 +72,15 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
}
- (void)removeAllData {
for (NSString *path in [self.fileManager subpathsAtPath:self.cachePath]) {
NSString *filePath = [self.cachePath stringByAppendingPathComponent:path];
[self.fileManager removeItemAtPath:filePath error:nil];
NSURL *srcURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:srcURL
includingPropertiesForKeys:@[]
options:(NSDirectoryEnumerationOptions)0
errorHandler:NULL];
for (NSURL *url in fileEnumerator) {
@autoreleasepool {
[self.fileManager removeItemAtURL:url error:nil];
}
}
}
@ -84,11 +90,28 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
- (void)removeExpiredData {
NSDate *expirationDate = [NSDate dateWithTimeIntervalSinceNow:-self.config.maxDiskAge];
for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) {
NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName];
NSDate *modificationDate = [[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileModificationDate];
if ([[modificationDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
[self.fileManager removeItemAtPath:filePath error:nil];
NSURL *diskCacheURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
NSArray<NSString *> *resourceKeys = @[NSURLIsDirectoryKey, NSURLAttributeModificationDateKey];
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
includingPropertiesForKeys:resourceKeys
options:NSDirectoryEnumerationSkipsHiddenFiles
errorHandler:NULL];
for (NSURL *fileURL in fileEnumerator) {
@autoreleasepool {
NSError *error;
NSDictionary<NSString *, id> *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error];
// Skip directories and errors.
if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) {
continue;;
}
// Remove files that are older than the expiration date;
NSDate *modifiedDate = resourceValues[NSURLAttributeModificationDateKey];
if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
[self.fileManager removeItemAtURL:fileURL error:nil];
}
}
}
}
@ -98,14 +121,29 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
}
- (NSUInteger)totalCount {
return [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count;
NSUInteger count = 0;
@autoreleasepool {
count = [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count;
}
return count;
}
- (NSUInteger)totalSize {
NSUInteger size = 0;
for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) {
NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName];
size += [[[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileSize] unsignedIntegerValue];
@autoreleasepool {
NSURL *pathURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL
includingPropertiesForKeys:@[NSURLFileSizeKey]
options:(NSDirectoryEnumerationOptions)0
errorHandler:NULL];
for (NSURL *fileURL in fileEnumerator) {
@autoreleasepool {
NSNumber *fileSize;
[fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL];
size += fileSize.unsignedIntegerValue;
}
}
}
return size;
}