Replace NSFileManager.enumeratorAtPath with enumeratorAtURL for performance and RAM saving (#3690)
* fix #3689 1. repalce @selector(enumeratorAtURL:) with @selector(enumeratorAtURL:) 2. replace ioQueueAttributes from DISPATCH_QUEUE_SERIAL to DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL >= iOS 10 * fix: remove NSDirectoryEnumerationProducesRelativePathURLs option * feat: replace enumeratorAtPath: * fix: update test44DiskCacheMigrationFromOldVersion --------- Co-authored-by: huangchengzhi <huangchengzhi@bytedance.com>
This commit is contained in:
ChengzhiHuang
GitHub
parent
b156318507
commit
d5e3e7f7c5
|
|
@ -165,7 +165,7 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
|
|||
NSArray<NSString *> *resourceKeys = @[NSURLIsDirectoryKey, cacheContentDateKey, NSURLTotalFileAllocatedSizeKey];
|
||||
|
||||
// This enumerator prefetches useful properties for our cache files.
|
||||
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
|
||||
includingPropertiesForKeys:resourceKeys
|
||||
options:NSDirectoryEnumerationSkipsHiddenFiles
|
||||
errorHandler:NULL];
|
||||
|
|
@ -180,6 +180,7 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
|
|||
// 2. Storing file attributes for the size-based cleanup pass.
|
||||
NSMutableArray<NSURL *> *urlsToDelete = [[NSMutableArray alloc] init];
|
||||
for (NSURL *fileURL in fileEnumerator) {
|
||||
@autoreleasepool {
|
||||
NSError *error;
|
||||
NSDictionary<NSString *, id> *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error];
|
||||
|
||||
|
|
@ -200,6 +201,7 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
|
|||
currentCacheSize += totalAllocatedSize.unsignedIntegerValue;
|
||||
cacheFiles[fileURL] = resourceValues;
|
||||
}
|
||||
}
|
||||
|
||||
for (NSURL *fileURL in urlsToDelete) {
|
||||
[self.fileManager removeItemAtURL:fileURL error:nil];
|
||||
|
|
@ -240,19 +242,37 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
|
|||
|
||||
- (NSUInteger)totalSize {
|
||||
NSUInteger size = 0;
|
||||
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath];
|
||||
for (NSString *fileName in fileEnumerator) {
|
||||
NSString *filePath = [self.diskCachePath stringByAppendingPathComponent:fileName];
|
||||
NSDictionary<NSString *, id> *attrs = [self.fileManager attributesOfItemAtPath:filePath error:nil];
|
||||
size += [attrs fileSize];
|
||||
|
||||
// Use URL-based enumerator instead of Path(NSString *)-based enumerator to reduce
|
||||
// those objects(ex. NSPathStore2/_NSCFString/NSConcreteData) created during traversal.
|
||||
// Even worse, those objects are added into AutoreleasePool, in background threads,
|
||||
// the time to release those objects is undifined(according to the usage of CPU)
|
||||
// It will truely consumes a lot of VM, up to cause OOMs.
|
||||
@autoreleasepool {
|
||||
NSURL *pathURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES];
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL
|
||||
includingPropertiesForKeys:@[NSURLFileSizeKey]
|
||||
options:(NSDirectoryEnumerationOptions)0
|
||||
errorHandler:NULL];
|
||||
|
||||
for (NSURL *fileURL in fileEnumerator) {
|
||||
@autoreleasepool {
|
||||
NSNumber *fileSize;
|
||||
[fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL];
|
||||
size += fileSize.unsignedIntegerValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
- (NSUInteger)totalCount {
|
||||
NSUInteger count = 0;
|
||||
NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath];
|
||||
@autoreleasepool {
|
||||
NSURL *diskCacheURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES];
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL includingPropertiesForKeys:@[] options:(NSDirectoryEnumerationOptions)0 errorHandler:nil];
|
||||
count = fileEnumerator.allObjects.count;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
|
@ -295,13 +315,21 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis
|
|||
}
|
||||
} else {
|
||||
// New directory exist, merge the files
|
||||
NSDirectoryEnumerator *dirEnumerator = [self.fileManager enumeratorAtPath:srcPath];
|
||||
NSString *file;
|
||||
while ((file = [dirEnumerator nextObject])) {
|
||||
[self.fileManager moveItemAtPath:[srcPath stringByAppendingPathComponent:file] toPath:[dstPath stringByAppendingPathComponent:file] error:nil];
|
||||
NSURL *srcURL = [NSURL fileURLWithPath:srcPath isDirectory:YES];
|
||||
NSDirectoryEnumerator<NSURL *> *srcDirEnumerator = [self.fileManager enumeratorAtURL:srcURL
|
||||
includingPropertiesForKeys:@[]
|
||||
options:(NSDirectoryEnumerationOptions)0
|
||||
errorHandler:NULL];
|
||||
for (NSURL *url in srcDirEnumerator) {
|
||||
@autoreleasepool {
|
||||
NSString *dstFilePath = [dstPath stringByAppendingPathComponent:url.lastPathComponent];
|
||||
NSURL *dstFileURL = [NSURL fileURLWithPath:dstFilePath isDirectory:NO];
|
||||
[self.fileManager moveItemAtURL:url toURL:dstFileURL error:nil];
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the old path
|
||||
[self.fileManager removeItemAtPath:srcPath error:nil];
|
||||
[self.fileManager removeItemAtURL:srcURL error:nil];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ typedef NS_ENUM(NSUInteger, SDImageCacheConfigExpireType) {
|
|||
|
||||
/**
|
||||
* The dispatch queue attr for ioQueue. You can config the QoS and concurrent/serial to internal IO queue. The ioQueue is used by SDImageCache to access read/write for disk data.
|
||||
* Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL), to use serial dispatch queue to ensure single access for disk data. It's safe but may be slow.
|
||||
* Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL) under iOS 10, `DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL` above and equal iOS 10, using serial dispatch queue is to ensure single access for disk data. It's safe but may be slow.
|
||||
* @note You can override this to use `DISPATCH_QUEUE_CONCURRENT`, use concurrent queue.
|
||||
* @warning **MAKE SURE** to keep `diskCacheWritingOptions` to use `NSDataWritingAtomic`, or concurrent queue may cause corrupted disk data (because multiple threads read/write same file without atomic is not IO-safe).
|
||||
* @note This value does not support dynamic changes. Which means further modification on this value after cache initialized has no effect.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,11 @@ static const NSInteger kDefaultCacheMaxDiskAge = 60 * 60 * 24 * 7; // 1 week
|
|||
_maxDiskSize = 0;
|
||||
_diskCacheExpireType = SDImageCacheConfigExpireTypeModificationDate;
|
||||
_fileManager = nil;
|
||||
if (@available(iOS 10.0, tvOS 10.0, macOS 10.12, watchOS 3.0, *)) {
|
||||
_ioQueueAttributes = DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL; // DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM
|
||||
} else {
|
||||
_ioQueueAttributes = DISPATCH_QUEUE_SERIAL; // NULL
|
||||
}
|
||||
_memoryCacheClass = [SDMemoryCache class];
|
||||
_diskCacheClass = [SDDiskCache class];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -559,18 +559,31 @@ static NSString *kTestImageKeyPNG = @"TestImageKey.png";
|
|||
NSFileManager *fileManager = [[NSFileManager alloc] init];
|
||||
config.fileManager = fileManager;
|
||||
|
||||
// Fake to store a.png into old path
|
||||
// Fake to store a%@.png into old path
|
||||
NSString *newDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"com.hackemist.SDImageCache"] stringByAppendingPathComponent:@"default"];
|
||||
NSString *oldDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"default"] stringByAppendingPathComponent:@"com.hackemist.SDWebImageCache.default"];
|
||||
[fileManager createDirectoryAtPath:oldDefaultPath withIntermediateDirectories:YES attributes:nil error:nil];
|
||||
[fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:@"a.png"] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil];
|
||||
|
||||
// Create 100 files to Migrate
|
||||
for (NSUInteger i = 0; i < 100; i++) {
|
||||
NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)];
|
||||
[fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:fileName] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil];
|
||||
}
|
||||
|
||||
// Call migration
|
||||
SDDiskCache *diskCache = [[SDDiskCache alloc] initWithCachePath:newDefaultPath config:config];
|
||||
[diskCache moveCacheDirectoryFromPath:oldDefaultPath toPath:newDefaultPath];
|
||||
|
||||
// Expect a.png into new path
|
||||
BOOL exist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:@"a.png"]];
|
||||
expect(exist).beTruthy();
|
||||
// Expect a%@.png into new path and oldDefaultPath is deleted
|
||||
BOOL isDirectory = NO;
|
||||
for (NSUInteger i = 0; i < 100; i++) {
|
||||
NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)];
|
||||
BOOL newFileExist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:fileName] isDirectory:&isDirectory];
|
||||
expect(newFileExist).beTruthy();
|
||||
expect(isDirectory).beFalsy();
|
||||
}
|
||||
BOOL oldDefaultPathExist = [fileManager fileExistsAtPath:oldDefaultPath];
|
||||
expect(oldDefaultPathExist).beFalsy();
|
||||
}
|
||||
|
||||
- (void)test45DiskCacheRemoveExpiredData {
|
||||
|
|
|
|||
|
|
@ -72,9 +72,15 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
|
|||
}
|
||||
|
||||
- (void)removeAllData {
|
||||
for (NSString *path in [self.fileManager subpathsAtPath:self.cachePath]) {
|
||||
NSString *filePath = [self.cachePath stringByAppendingPathComponent:path];
|
||||
[self.fileManager removeItemAtPath:filePath error:nil];
|
||||
NSURL *srcURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:srcURL
|
||||
includingPropertiesForKeys:@[]
|
||||
options:(NSDirectoryEnumerationOptions)0
|
||||
errorHandler:NULL];
|
||||
for (NSURL *url in fileEnumerator) {
|
||||
@autoreleasepool {
|
||||
[self.fileManager removeItemAtURL:url error:nil];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -84,11 +90,28 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
|
|||
|
||||
- (void)removeExpiredData {
|
||||
NSDate *expirationDate = [NSDate dateWithTimeIntervalSinceNow:-self.config.maxDiskAge];
|
||||
for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) {
|
||||
NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName];
|
||||
NSDate *modificationDate = [[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileModificationDate];
|
||||
if ([[modificationDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
|
||||
[self.fileManager removeItemAtPath:filePath error:nil];
|
||||
NSURL *diskCacheURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
|
||||
NSArray<NSString *> *resourceKeys = @[NSURLIsDirectoryKey, NSURLAttributeModificationDateKey];
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL
|
||||
includingPropertiesForKeys:resourceKeys
|
||||
options:NSDirectoryEnumerationSkipsHiddenFiles
|
||||
errorHandler:NULL];
|
||||
|
||||
for (NSURL *fileURL in fileEnumerator) {
|
||||
@autoreleasepool {
|
||||
NSError *error;
|
||||
NSDictionary<NSString *, id> *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error];
|
||||
|
||||
// Skip directories and errors.
|
||||
if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) {
|
||||
continue;;
|
||||
}
|
||||
|
||||
// Remove files that are older than the expiration date;
|
||||
NSDate *modifiedDate = resourceValues[NSURLAttributeModificationDateKey];
|
||||
if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) {
|
||||
[self.fileManager removeItemAtURL:fileURL error:nil];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -98,14 +121,29 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac
|
|||
}
|
||||
|
||||
- (NSUInteger)totalCount {
|
||||
return [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count;
|
||||
NSUInteger count = 0;
|
||||
@autoreleasepool {
|
||||
count = [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
- (NSUInteger)totalSize {
|
||||
NSUInteger size = 0;
|
||||
for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) {
|
||||
NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName];
|
||||
size += [[[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileSize] unsignedIntegerValue];
|
||||
@autoreleasepool {
|
||||
NSURL *pathURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES];
|
||||
NSDirectoryEnumerator<NSURL *> *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL
|
||||
includingPropertiesForKeys:@[NSURLFileSizeKey]
|
||||
options:(NSDirectoryEnumerationOptions)0
|
||||
errorHandler:NULL];
|
||||
|
||||
for (NSURL *fileURL in fileEnumerator) {
|
||||
@autoreleasepool {
|
||||
NSNumber *fileSize;
|
||||
[fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL];
|
||||
size += fileSize.unsignedIntegerValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue