ZcashLightClientKit/Tests/OfflineTests/FsBlockStorageTests.swift

//
//  FsBlockStorageTests.swift
//  
//
//  Created by Francisco Gindre on 12/15/22.
//
import XCTest
@testable import TestUtils
@testable import ZcashLightClientKit

var logger = OSLogger(logLevel: .debug)

final class FsBlockStorageTests: XCTestCase {
    let testTempDirectory = URL(fileURLWithPath: NSString(
        string: NSTemporaryDirectory()
    )
        .appendingPathComponent("tmp-\(Int.random(in: 0 ... .max))"))

    let testFileManager = FileManager()

    var fsBlockDb: URL!

    override func setUpWithError() throws {
        try super.setUpWithError()
        // Put setup code here. This method is called before the invocation of each test method in the class.
        self.fsBlockDb = self.testTempDirectory.appendingPathComponent("FsBlockDb-\(Int.random(in: 0 ... .max))")
        try self.testFileManager.createDirectory(at: self.testTempDirectory, withIntermediateDirectories: false)
        try self.testFileManager.createDirectory(at: self.fsBlockDb, withIntermediateDirectories: false)
    }

    override func tearDownWithError() throws {
        try super.tearDownWithError()
        try? testFileManager.removeItem(at: testTempDirectory)
    }

    func testLatestHeightEmptyCache() throws {
        let emptyCache: FSCompactBlockRepository = .emptyTemporaryCache

        try emptyCache.create()

        XCTAssertEqual(emptyCache.latestHeight(), .empty())
    }

    func testRewindEmptyCacheDoesNothing() throws {
        let emptyCache: FSCompactBlockRepository = .emptyTemporaryCache

        try emptyCache.create()

        try emptyCache.rewind(to: 1000000)
    }

    func testRewindEmptyCacheDoesNothingAsync() async throws {
        let emptyCache: FSCompactBlockRepository = .emptyTemporaryCache

        try emptyCache.create()

        try await emptyCache.rewindAsync(to: 1000000)
    }

    func testWhenBlockIsStoredItFollowsTheDescribedFormat() async throws {
        let blockNameFixture = "This-is-a-fixture"

        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: self.testTempDirectory,
            metadataStore: .mock,
            blockDescriptor: ZcashCompactBlockDescriptor(
                height: { _ in nil },
                describe: { _ in blockNameFixture },
                compare: { _, _ in nil }
            ),
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try freshCache.create()

        let fakeBlock = StubBlockCreator.createRandomDataBlock(with: 1234)!

        try await freshCache.write(blocks: [fakeBlock])

        let blockFilename = freshCache.blocksDirectory
            .appendingPathComponent(blockNameFixture)
            .path

        XCTAssertTrue(FileManager.default.isReadableFile(atPath: blockFilename))
    }

    func testWhenBlockIsStoredItFollowsTheFilenameConvention() async throws {
        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .mock,
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try freshCache.create()

        let fakeBlock = StubBlockCreator.createRandomDataBlock(with: 1234)!
        let fakeBlockHash = fakeBlock.meta.hash.toHexStringTxId()

        try await freshCache.write(blocks: [fakeBlock])

        let blockFilename = freshCache.blocksDirectory
            .appendingPathComponent(
                "\(1234)-\(fakeBlockHash)-compactblock"
            )
            .path

        XCTAssertTrue(FileManager.default.isReadableFile(atPath: blockFilename))
    }

    func testRewindDeletesTheRightBlocks() async throws {
        let contentProvider = DirectoryListingProviders.defaultSorted
        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .mock,
            blockDescriptor: .live,
            contentProvider: contentProvider
        )

        try freshCache.create()

        let blockRange = CompactBlockRange(uncheckedBounds: (1000, 2000))

        let fakeBlocks = StubBlockCreator.createBlockRange(blockRange)!

        let rewindHeight = BlockHeight(1500)

        try await freshCache.write(blocks: fakeBlocks)

        let contents = try contentProvider.listContents(of: freshCache.blocksDirectory)

        XCTAssertEqual(contents.count, blockRange.count)

        guard let firstStoredBlock = contents.first else {
            XCTFail("no stored block")
            return
        }

        guard let filename = try firstStoredBlock.resourceValues(forKeys: [URLResourceKey.nameKey]).name else {
            XCTFail("no filename")
            return
        }
        
        XCTAssertEqual(ZcashCompactBlockDescriptor.live.height(filename), 1000)

        guard let lastStoredBlock = contents.last else {
            XCTFail("no stored block")
            return
        }

        XCTAssertEqual(ZcashCompactBlockDescriptor.live.height(lastStoredBlock.lastPathComponent), 2000)

        try await freshCache.rewindAsync(to: rewindHeight)

        let afterRewindContents = try contentProvider.listContents(of: freshCache.blocksDirectory)

        XCTAssertEqual(afterRewindContents.count, 501)

        guard let firstStoredBlockAfterRewind = afterRewindContents.first else {
            XCTFail("no stored block")
            return
        }

        XCTAssertEqual(ZcashCompactBlockDescriptor.live.height(firstStoredBlockAfterRewind.lastPathComponent), 1000)

        guard let lastStoredBlockAfterRewind = afterRewindContents.last else {
            XCTFail("no stored block")
            return
        }

        XCTAssertEqual(ZcashCompactBlockDescriptor.live.height(lastStoredBlockAfterRewind.lastPathComponent), 1500)
    }

    func testGetLatestHeight() async throws {
        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: ZcashRustBackend.self),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try freshCache.create()

        let blockRange = CompactBlockRange(uncheckedBounds: (1000, 2000))

        let fakeBlocks = StubBlockCreator.createBlockRange(blockRange)!

        try await freshCache.write(blocks: fakeBlocks)

        let latestHeight = freshCache.latestHeight()

        XCTAssertEqual(latestHeight, 2000)
    }

    func testBlockDescriptorFiltersBlocksGreaterThan() throws {
        let cacheList = [
            "1000-DEADBEEF-block",
            "1001-DEADBEEF1-block",
            "1002-DEADBEEF2-block",
            "1003-DEADBEEF3-block",
            "1004-DEADBEEF4-block",
            "1005-DEADBEEF5-block",
            "1006-DEADBEEF6-block",
            "1007-DEADBEEF7-block",
            "1008-DEADBEEF8-block",
            "1009-DEADBEEF9-block",
            "1010-DEADBEEFA-block"
        ]

        XCTAssertEqual(
            try cacheList.filter { try $0.filterGreaterThan(1006, with: .live) },
            [
                "1007-DEADBEEF7-block",
                "1008-DEADBEEF8-block",
                "1009-DEADBEEF9-block",
                "1010-DEADBEEFA-block"
            ]
        )
    }

    func testBlockDescriptorFiltersThrowsIfFileDescriptorFails() throws {
        let cacheList = [
            "1000-DEADBEEF-block",
            "1001-DEADBEEF1-block",
            "1002-DEADBEEF2-block",
            "1003-DEADBEEF3-block",
            "1004-DEADBEEF4-block",
            "1005-DEADBEEF5-block",
            "a-DEADBEEF6-block",
            "1007-DEADBEEF7-block",
            "1008-DEADBEEF8-block",
            "1009-DEADBEEF9-block",
            "1010-DEADBEEFA-block"
        ]

        XCTAssertThrowsError(try cacheList.filter { try $0.filterGreaterThan(1006, with: .live) })
    }

    func testRewindBlockSelectTheProperFilesByName() throws {
        let cacheList = try [
            "1000-DEADBEEF-block",
            "1001-DEADBEEF1-block",
            "1002-DEADBEEF2-block",
            "1003-DEADBEEF3-block",
            "1004-DEADBEEF4-block",
            "1005-DEADBEEF5-block",
            "1006-DEADBEEF6-block",
            "1007-DEADBEEF7-block",
            "1008-DEADBEEF8-block",
            "1009-DEADBEEF9-block",
            "1010-DEADBEEFA-block"
        ].map { filename in
            var url = self.fsBlockDb.appendingPathComponent(filename)
            guard self.testFileManager.createFile(atPath: url.path, contents: nil) else {
                XCTFail("couldn't create file at \(url.absoluteString)")
                throw CompactBlockRepositoryError.malformedCacheEntry("couldn't create file at \(url.path)")
            }
            var resourceValues = URLResourceValues()
            resourceValues.name = filename

            try url.setResourceValues(resourceValues)

            return url
        }

        let expectedDeleteList = try [
            "1007-DEADBEEF7-block",
            "1008-DEADBEEF8-block",
            "1009-DEADBEEF9-block",
            "1010-DEADBEEFA-block"
        ].reversed().map { filename in
            var url = self.fsBlockDb.appendingPathComponent(filename)
            var resourceValues = URLResourceValues()
            resourceValues.name = filename
            try url.setResourceValues(resourceValues)
            return url
        }

        XCTAssertEqual(
            try FSCompactBlockRepository.filterBlockFiles(from: cacheList, toRewind: 1006, with: .live),
            expectedDeleteList
        )
    }

    func testClearTheCache() async throws {
        let fsBlockCache = FSCompactBlockRepository(
            fsBlockDbRoot: self.testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: ZcashRustBackend.self),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.naive
        )

        try fsBlockCache.create()

        guard let stubBlocks = StubBlockCreator.createBlockRange(1000 ... 1010) else {
            XCTFail("Something Happened. Creating Stub blocks failed")
            return
        }

        try await fsBlockCache.write(blocks: stubBlocks)
        XCTAssertEqual(fsBlockCache.latestHeight(), 1010)

        try await fsBlockCache.clear()

        XCTAssertEqual(fsBlockCache.latestHeight(), .empty())
    }

    func testCreateDoesntFailWhenAlreadyCreated() throws {
        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .mock,
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try freshCache.create()

        XCTAssertNoThrow(try freshCache.create())
    }

    func testStoringTenSandblastedBlocks() async throws {
        let realRustBackend = ZcashRustBackend.self

        let realCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: realRustBackend),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try realCache.create()

        guard let sandblastedBlocks = try SandblastSimulator().sandblast(with: CompactBlockRange(uncheckedBounds: (10, 19))) else {
            XCTFail("failed to create sandblasted blocks")
            return
        }

        let startTime = Date()
        try await realCache.write(blocks: sandblastedBlocks)
        let endTime = Date()

        let timePassed = startTime.distance(to: endTime)

        XCTAssertLessThan(timePassed, 0.5)

        let latestHeight = await realCache.latestHeightAsync()

        XCTAssertEqual(latestHeight, 19)
    }

    func testStoringTenSandblastedBlocksFailsAndThrows() async throws {
        let realRustBackend = ZcashRustBackend.self

        let realCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: realRustBackend),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted,
            fileWriter: FSBlockFileWriter(writeToURL: { _, _ in  throw FixtureError.arbitraryError })
        )

        try realCache.create()

        guard let sandblastedBlocks = try SandblastSimulator().sandblast(with: CompactBlockRange(uncheckedBounds: (10, 19))) else {
            XCTFail("failed to create sandblasted blocks")
            return
        }

        do {
            try await realCache.write(blocks: sandblastedBlocks)
            XCTFail("This call should have failed")
        } catch {
            XCTAssertEqual(error as? CompactBlockRepositoryError, CompactBlockRepositoryError.failedToWriteBlock(sandblastedBlocks[0]))
        }
    }

    func testStoringTenSandblastedBlocksAndRewindFiveThenStoreThemBack() async throws {
        let realRustBackend = ZcashRustBackend.self

        let realCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: realRustBackend),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try realCache.create()

        guard let sandblastedBlocks = try SandblastSimulator().sandblast(with: CompactBlockRange(uncheckedBounds: (10, 19))) else {
            XCTFail("failed to create sandblasted blocks")
            return
        }

        let startTime = Date()
        try await realCache.write(blocks: sandblastedBlocks)
        let endTime = Date()

        let timePassed = startTime.distance(to: endTime)

        XCTAssertLessThan(timePassed, 0.5)

        let latestHeight = await realCache.latestHeightAsync()

        XCTAssertEqual(latestHeight, 19)

        try realCache.rewind(to: 14)

        let rewoundHeight = await realCache.latestHeightAsync()

        XCTAssertEqual(rewoundHeight, 14)

        let blockSlice = [ZcashCompactBlock](sandblastedBlocks[5...])
        try await realCache.write(blocks: blockSlice)

        let newLatestHeight = await realCache.latestHeightAsync()

        XCTAssertEqual(newLatestHeight, 19)
    }

    func testMetadataStoreThrowsWhenCallReturnsFalse() async {
        guard let sandblastedBlocks = try? SandblastSimulator().sandblast(with: CompactBlockRange(uncheckedBounds: (10, 19))) else {
            XCTFail("failed to create sandblasted blocks")
            return
        }

        MockRustBackend.writeBlocksMetadataResult = { false }

        do {
            try await FSMetadataStore.saveBlocksMeta(sandblastedBlocks, fsBlockDbRoot: testTempDirectory, rustBackend: MockRustBackend.self)
        } catch CompactBlockRepositoryError.failedToWriteMetadata {
            // this is fine
        } catch {
            XCTFail("Expected `CompactBlockRepositoryError.failedToWriteMetadata` but found: \(error.localizedDescription)")
        }
    }

    func testMetadataStoreThrowsWhenRustThrows() async {
        guard let sandblastedBlocks = try? SandblastSimulator().sandblast(with: CompactBlockRange(uncheckedBounds: (10, 19))) else {
            XCTFail("failed to create sandblasted blocks")
            return
        }

        MockRustBackend.writeBlocksMetadataResult = { throw RustWeldingError.genericError(message: "oops") }

        do {
            try await FSMetadataStore.saveBlocksMeta(sandblastedBlocks, fsBlockDbRoot: testTempDirectory, rustBackend: MockRustBackend.self)
        } catch CompactBlockRepositoryError.failedToWriteMetadata {
            // this is fine
        } catch {
            XCTFail("Expected `CompactBlockRepositoryError.failedToWriteMetadata` but found: \(error.localizedDescription)")
        }
    }

    func testMetadataStoreThrowsWhenRewindFails() {
        MockRustBackend.rewindCacheToHeightResult = { false }
        let expectedHeight = BlockHeight(1000)

        XCTAssertThrowsError(
            try FSMetadataStore.live(
                fsBlockDbRoot: testTempDirectory,
                rustBackend: MockRustBackend.self
            )
            .rewindToHeight(expectedHeight)
        ) { error in
            guard let repositoryError = error as? CompactBlockRepositoryError else {
                XCTFail("Expected CompactBlockRepositoryError. Found \(error)")
                return
            }

            switch repositoryError {
            case .failedToRewind(let height):
                XCTAssertEqual(height, expectedHeight)
            default:
                XCTFail("Expected `CompactBlockRepositoryError.failedToRewind`. Found \(error)")
            }
        }
    }

    // Disabled for now becasue we are not getting consistent results on GA Ci
    func disable_testPerformanceExample() async throws {
        // NOTE: performance tests don't work with async code. Thanks Apple!
        let freshCache = FSCompactBlockRepository(
            fsBlockDbRoot: testTempDirectory,
            metadataStore: .live(fsBlockDbRoot: testTempDirectory, rustBackend: ZcashRustBackend.self),
            blockDescriptor: .live,
            contentProvider: DirectoryListingProviders.defaultSorted
        )

        try freshCache.create()

        let blockRange = CompactBlockRange(uncheckedBounds: (1000, 2000))

        let fakeBlocks = try SandblastSimulator().sandblast(with: blockRange)!

        let startTime = Date()
        try await freshCache.write(blocks: fakeBlocks)
        let endTime = Date()

        let latestHeight = await freshCache.latestHeightAsync()

        XCTAssertEqual(latestHeight, 2000)

        let total = startTime.distance(to: endTime)
        //        let totalKiloBytes = fakeBlocks.map { $0.data.count }.reduce(0, +) / 1024 // 245055
        XCTAssertGreaterThan(1.5, total)
    }
}

extension FSCompactBlockRepository {
    static var emptyTemporaryCache: FSCompactBlockRepository {
        FSCompactBlockRepository(
            fsBlockDbRoot: URL(fileURLWithPath: NSString(
                string: NSTemporaryDirectory()
            ).appendingPathComponent("tmp-\(Int.random(in: 0 ... .max))")),
            metadataStore: .mock,
            blockDescriptor: ZcashCompactBlockDescriptor(
                height: { _ in BlockHeight() },
                describe: { _ in "123456-deadbeef-block" },
                compare: { _, _ in nil }
            ),
            contentProvider: SortedDirectoryContentProvider(
                fileManager: FileManager.default,
                sorting: { _, _ in false }
            )
        )
    }
}

enum FixtureError: Error, Equatable {
    case arbitraryError
}

extension FSMetadataStore {
    static let mock = FSMetadataStore { _ in }
    rewindToHeight: { _ in }
    initFsBlockDbRoot: { _ in true }
    latestHeightAsync: { .empty() }
    latestHeight: {
        .empty()
    }
}