diff --git a/SDWebImage/Core/SDDiskCache.m b/SDWebImage/Core/SDDiskCache.m index 6e9dfaab..82eb0d6b 100644 --- a/SDWebImage/Core/SDDiskCache.m +++ b/SDWebImage/Core/SDDiskCache.m @@ -165,7 +165,7 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis NSArray *resourceKeys = @[NSURLIsDirectoryKey, cacheContentDateKey, NSURLTotalFileAllocatedSizeKey]; // This enumerator prefetches useful properties for our cache files. - NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL includingPropertiesForKeys:resourceKeys options:NSDirectoryEnumerationSkipsHiddenFiles errorHandler:NULL]; @@ -180,25 +180,27 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis // 2. Storing file attributes for the size-based cleanup pass. NSMutableArray *urlsToDelete = [[NSMutableArray alloc] init]; for (NSURL *fileURL in fileEnumerator) { - NSError *error; - NSDictionary *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error]; - - // Skip directories and errors. - if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) { - continue; + @autoreleasepool { + NSError *error; + NSDictionary *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error]; + + // Skip directories and errors. + if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) { + continue; + } + + // Remove files that are older than the expiration date; + NSDate *modifiedDate = resourceValues[cacheContentDateKey]; + if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) { + [urlsToDelete addObject:fileURL]; + continue; + } + + // Store a reference to this file and account for its total size. + NSNumber *totalAllocatedSize = resourceValues[NSURLTotalFileAllocatedSizeKey]; + currentCacheSize += totalAllocatedSize.unsignedIntegerValue; + cacheFiles[fileURL] = resourceValues; } - - // Remove files that are older than the expiration date; - NSDate *modifiedDate = resourceValues[cacheContentDateKey]; - if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) { - [urlsToDelete addObject:fileURL]; - continue; - } - - // Store a reference to this file and account for its total size. - NSNumber *totalAllocatedSize = resourceValues[NSURLTotalFileAllocatedSizeKey]; - currentCacheSize += totalAllocatedSize.unsignedIntegerValue; - cacheFiles[fileURL] = resourceValues; } for (NSURL *fileURL in urlsToDelete) { @@ -240,19 +242,37 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis - (NSUInteger)totalSize { NSUInteger size = 0; - NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath]; - for (NSString *fileName in fileEnumerator) { - NSString *filePath = [self.diskCachePath stringByAppendingPathComponent:fileName]; - NSDictionary *attrs = [self.fileManager attributesOfItemAtPath:filePath error:nil]; - size += [attrs fileSize]; + + // Use URL-based enumerator instead of Path(NSString *)-based enumerator to reduce + // those objects(ex. NSPathStore2/_NSCFString/NSConcreteData) created during traversal. + // Even worse, those objects are added into AutoreleasePool, in background threads, + // the time to release those objects is undifined(according to the usage of CPU) + // It will truely consumes a lot of VM, up to cause OOMs. + @autoreleasepool { + NSURL *pathURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES]; + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL + includingPropertiesForKeys:@[NSURLFileSizeKey] + options:(NSDirectoryEnumerationOptions)0 + errorHandler:NULL]; + + for (NSURL *fileURL in fileEnumerator) { + @autoreleasepool { + NSNumber *fileSize; + [fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL]; + size += fileSize.unsignedIntegerValue; + } + } } return size; } - (NSUInteger)totalCount { NSUInteger count = 0; - NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtPath:self.diskCachePath]; - count = fileEnumerator.allObjects.count; + @autoreleasepool { + NSURL *diskCacheURL = [NSURL fileURLWithPath:self.diskCachePath isDirectory:YES]; + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL includingPropertiesForKeys:@[] options:(NSDirectoryEnumerationOptions)0 errorHandler:nil]; + count = fileEnumerator.allObjects.count; + } return count; } @@ -295,13 +315,21 @@ static NSString * const SDDiskCacheExtendedAttributeName = @"com.hackemist.SDDis } } else { // New directory exist, merge the files - NSDirectoryEnumerator *dirEnumerator = [self.fileManager enumeratorAtPath:srcPath]; - NSString *file; - while ((file = [dirEnumerator nextObject])) { - [self.fileManager moveItemAtPath:[srcPath stringByAppendingPathComponent:file] toPath:[dstPath stringByAppendingPathComponent:file] error:nil]; + NSURL *srcURL = [NSURL fileURLWithPath:srcPath isDirectory:YES]; + NSDirectoryEnumerator *srcDirEnumerator = [self.fileManager enumeratorAtURL:srcURL + includingPropertiesForKeys:@[] + options:(NSDirectoryEnumerationOptions)0 + errorHandler:NULL]; + for (NSURL *url in srcDirEnumerator) { + @autoreleasepool { + NSString *dstFilePath = [dstPath stringByAppendingPathComponent:url.lastPathComponent]; + NSURL *dstFileURL = [NSURL fileURLWithPath:dstFilePath isDirectory:NO]; + [self.fileManager moveItemAtURL:url toURL:dstFileURL error:nil]; + } } + // Remove the old path - [self.fileManager removeItemAtPath:srcPath error:nil]; + [self.fileManager removeItemAtURL:srcURL error:nil]; } } diff --git a/SDWebImage/Core/SDImageCacheConfig.h b/SDWebImage/Core/SDImageCacheConfig.h index 7e53ac7d..91889158 100644 --- a/SDWebImage/Core/SDImageCacheConfig.h +++ b/SDWebImage/Core/SDImageCacheConfig.h @@ -129,7 +129,7 @@ typedef NS_ENUM(NSUInteger, SDImageCacheConfigExpireType) { /** * The dispatch queue attr for ioQueue. You can config the QoS and concurrent/serial to internal IO queue. The ioQueue is used by SDImageCache to access read/write for disk data. - * Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL), to use serial dispatch queue to ensure single access for disk data. It's safe but may be slow. + * Defaults we use `DISPATCH_QUEUE_SERIAL`(NULL) under iOS 10, `DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL` above and equal iOS 10, using serial dispatch queue is to ensure single access for disk data. It's safe but may be slow. * @note You can override this to use `DISPATCH_QUEUE_CONCURRENT`, use concurrent queue. * @warning **MAKE SURE** to keep `diskCacheWritingOptions` to use `NSDataWritingAtomic`, or concurrent queue may cause corrupted disk data (because multiple threads read/write same file without atomic is not IO-safe). * @note This value does not support dynamic changes. Which means further modification on this value after cache initialized has no effect. diff --git a/SDWebImage/Core/SDImageCacheConfig.m b/SDWebImage/Core/SDImageCacheConfig.m index ee6db59d..6e594eda 100644 --- a/SDWebImage/Core/SDImageCacheConfig.m +++ b/SDWebImage/Core/SDImageCacheConfig.m @@ -36,7 +36,11 @@ static const NSInteger kDefaultCacheMaxDiskAge = 60 * 60 * 24 * 7; // 1 week _maxDiskSize = 0; _diskCacheExpireType = SDImageCacheConfigExpireTypeModificationDate; _fileManager = nil; - _ioQueueAttributes = DISPATCH_QUEUE_SERIAL; // NULL + if (@available(iOS 10.0, tvOS 10.0, macOS 10.12, watchOS 3.0, *)) { + _ioQueueAttributes = DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL; // DISPATCH_AUTORELEASE_FREQUENCY_WORK_ITEM + } else { + _ioQueueAttributes = DISPATCH_QUEUE_SERIAL; // NULL + } _memoryCacheClass = [SDMemoryCache class]; _diskCacheClass = [SDDiskCache class]; } diff --git a/Tests/Tests/SDImageCacheTests.m b/Tests/Tests/SDImageCacheTests.m index 599153d5..147debd0 100644 --- a/Tests/Tests/SDImageCacheTests.m +++ b/Tests/Tests/SDImageCacheTests.m @@ -559,18 +559,31 @@ static NSString *kTestImageKeyPNG = @"TestImageKey.png"; NSFileManager *fileManager = [[NSFileManager alloc] init]; config.fileManager = fileManager; - // Fake to store a.png into old path + // Fake to store a%@.png into old path NSString *newDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"com.hackemist.SDImageCache"] stringByAppendingPathComponent:@"default"]; NSString *oldDefaultPath = [[[self userCacheDirectory] stringByAppendingPathComponent:@"default"] stringByAppendingPathComponent:@"com.hackemist.SDWebImageCache.default"]; [fileManager createDirectoryAtPath:oldDefaultPath withIntermediateDirectories:YES attributes:nil error:nil]; - [fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:@"a.png"] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil]; + + // Create 100 files to Migrate + for (NSUInteger i = 0; i < 100; i++) { + NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)]; + [fileManager createFileAtPath:[oldDefaultPath stringByAppendingPathComponent:fileName] contents:[NSData dataWithContentsOfFile:[self testPNGPath]] attributes:nil]; + } + // Call migration SDDiskCache *diskCache = [[SDDiskCache alloc] initWithCachePath:newDefaultPath config:config]; [diskCache moveCacheDirectoryFromPath:oldDefaultPath toPath:newDefaultPath]; - // Expect a.png into new path - BOOL exist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:@"a.png"]]; - expect(exist).beTruthy(); + // Expect a%@.png into new path and oldDefaultPath is deleted + BOOL isDirectory = NO; + for (NSUInteger i = 0; i < 100; i++) { + NSString *fileName = [NSString stringWithFormat:@"a%@.png", @(i)]; + BOOL newFileExist = [fileManager fileExistsAtPath:[newDefaultPath stringByAppendingPathComponent:fileName] isDirectory:&isDirectory]; + expect(newFileExist).beTruthy(); + expect(isDirectory).beFalsy(); + } + BOOL oldDefaultPathExist = [fileManager fileExistsAtPath:oldDefaultPath]; + expect(oldDefaultPathExist).beFalsy(); } - (void)test45DiskCacheRemoveExpiredData { diff --git a/Tests/Tests/SDWebImageTestCache.m b/Tests/Tests/SDWebImageTestCache.m index f709f4eb..336c1048 100644 --- a/Tests/Tests/SDWebImageTestCache.m +++ b/Tests/Tests/SDWebImageTestCache.m @@ -72,9 +72,15 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac } - (void)removeAllData { - for (NSString *path in [self.fileManager subpathsAtPath:self.cachePath]) { - NSString *filePath = [self.cachePath stringByAppendingPathComponent:path]; - [self.fileManager removeItemAtPath:filePath error:nil]; + NSURL *srcURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES]; + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:srcURL + includingPropertiesForKeys:@[] + options:(NSDirectoryEnumerationOptions)0 + errorHandler:NULL]; + for (NSURL *url in fileEnumerator) { + @autoreleasepool { + [self.fileManager removeItemAtURL:url error:nil]; + } } } @@ -84,11 +90,28 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac - (void)removeExpiredData { NSDate *expirationDate = [NSDate dateWithTimeIntervalSinceNow:-self.config.maxDiskAge]; - for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) { - NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName]; - NSDate *modificationDate = [[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileModificationDate]; - if ([[modificationDate laterDate:expirationDate] isEqualToDate:expirationDate]) { - [self.fileManager removeItemAtPath:filePath error:nil]; + NSURL *diskCacheURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES]; + NSArray *resourceKeys = @[NSURLIsDirectoryKey, NSURLAttributeModificationDateKey]; + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:diskCacheURL + includingPropertiesForKeys:resourceKeys + options:NSDirectoryEnumerationSkipsHiddenFiles + errorHandler:NULL]; + + for (NSURL *fileURL in fileEnumerator) { + @autoreleasepool { + NSError *error; + NSDictionary *resourceValues = [fileURL resourceValuesForKeys:resourceKeys error:&error]; + + // Skip directories and errors. + if (error || !resourceValues || [resourceValues[NSURLIsDirectoryKey] boolValue]) { + continue;; + } + + // Remove files that are older than the expiration date; + NSDate *modifiedDate = resourceValues[NSURLAttributeModificationDateKey]; + if (expirationDate && [[modifiedDate laterDate:expirationDate] isEqualToDate:expirationDate]) { + [self.fileManager removeItemAtURL:fileURL error:nil]; + } } } } @@ -98,14 +121,29 @@ static NSString * const SDWebImageTestDiskCacheExtendedAttributeName = @"com.hac } - (NSUInteger)totalCount { - return [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count; + NSUInteger count = 0; + @autoreleasepool { + count = [self.fileManager contentsOfDirectoryAtPath:self.cachePath error:nil].count; + } + return count; } - (NSUInteger)totalSize { NSUInteger size = 0; - for (NSString *fileName in [self.fileManager enumeratorAtPath:self.cachePath]) { - NSString *filePath = [self.cachePath stringByAppendingPathComponent:fileName]; - size += [[[self.fileManager attributesOfItemAtPath:filePath error:nil] objectForKey:NSFileSize] unsignedIntegerValue]; + @autoreleasepool { + NSURL *pathURL = [NSURL fileURLWithPath:self.cachePath isDirectory:YES]; + NSDirectoryEnumerator *fileEnumerator = [self.fileManager enumeratorAtURL:pathURL + includingPropertiesForKeys:@[NSURLFileSizeKey] + options:(NSDirectoryEnumerationOptions)0 + errorHandler:NULL]; + + for (NSURL *fileURL in fileEnumerator) { + @autoreleasepool { + NSNumber *fileSize; + [fileURL getResourceValue:&fileSize forKey:NSURLFileSizeKey error:NULL]; + size += fileSize.unsignedIntegerValue; + } + } } return size; }