Feature: calculate file MD5 & SHA1 checksums (#630)

src/modules/candidateCombiner.ts

@@ -48,7 +48,7 @@ export default class CandidateCombiner extends Module {
 
     const game = CandidateCombiner.buildGame(dat, parentsToCandidates);
     const parent = new Parent(game.getName(), [game]);
-    const releaseCandidate = await CandidateCombiner.buildReleaseCandidate(
+    const releaseCandidate = CandidateCombiner.buildReleaseCandidate(
       dat,
       game,
       parentsToCandidates,
@@ -80,15 +80,15 @@ export default class CandidateCombiner extends Module {
     });
   }
 
-  private static async buildReleaseCandidate(
+  private static buildReleaseCandidate(
     dat: DAT,
     game: Game,
     parentsToCandidates: Map<Parent, ReleaseCandidate[]>,
-  ): Promise<ReleaseCandidate> {
-    const romsWithFiles = await Promise.all([...parentsToCandidates.values()]
+  ): ReleaseCandidate {
+    const romsWithFiles = [...parentsToCandidates.values()]
       .flatMap((releaseCandidates) => releaseCandidates)
       .flatMap((releaseCandidate) => releaseCandidate.getRomsWithFiles()
-        .map(async (romWithFiles) => {
+        .map((romWithFiles) => {
           // If the output isn't an archive then it must have been excluded (e.g. --zip-exclude),
           //  don't manipulate it.
           const outputFile = romWithFiles.getOutputFile();
@@ -101,7 +101,7 @@ export default class CandidateCombiner extends Module {
 
           // If the game has multiple ROMs, then group them in a folder in the archive
           if (releaseCandidate.getGame().getRoms().length > 1) {
-            outputEntry = await outputEntry.withEntryPath(path.join(
+            outputEntry = outputEntry.withEntryPath(path.join(
               releaseCandidate.getGame().getName(),
               outputEntry.getEntryPath(),
             ));
@@ -112,7 +112,7 @@ export default class CandidateCombiner extends Module {
             romWithFiles.getInputFile(),
             outputEntry,
           );
-        })));
+        }));
 
     return new ReleaseCandidate(game, undefined, romsWithFiles);
   }

src/modules/candidateGenerator.ts

@@ -273,6 +273,7 @@ export default class CandidateGenerator extends Module {
     if (inputFile.getFileHeader()
       && this.options.canRemoveHeader(dat, path.extname(outputPathParsed.entryPath))
     ) {
+      // TODO(cemmer): inputFile.getSizeWithoutHeader() ?
       outputFileCrc = inputFile.getCrc32WithoutHeader();
       outputFileSize = inputFile.getSizeWithoutHeader();
     }
@@ -284,14 +285,16 @@ export default class CandidateGenerator extends Module {
         new Zip(outputFilePath),
         outputPathParsed.entryPath,
         outputFileSize,
-        outputFileCrc,
+        // TODO(cemmer): calculate MD5 and SHA1 for testing purposes?
+        { crc32: outputFileCrc },
       );
     }
     // Otherwise, return a raw file
     return File.fileOf(
       outputFilePath,
       outputFileSize,
-      outputFileCrc,
+      // TODO(cemmer): calculate MD5 and SHA1 for testing purposes?
+      { crc32: outputFileCrc },
     );
   }
 

src/modules/candidatePatchGenerator.ts

@@ -9,6 +9,7 @@ import Release from '../types/dats/release.js';
 import ROM from '../types/dats/rom.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
 import File from '../types/files/file.js';
+import { ChecksumBitmask } from '../types/files/fileChecksums.js';
 import Options from '../types/options.js';
 import Patch from '../types/patches/patch.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
@@ -151,7 +152,8 @@ export default class CandidatePatchGenerator extends Module {
                   ? extractedFileName
                   : outputFile.getEntryPath(),
                 patch.getSizeAfter() ?? 0,
-                patch.getCrcAfter() ?? '00000000',
+                { crc32: patch.getCrcAfter() },
+                ChecksumBitmask.NONE,
                 outputFile.getFileHeader(),
                 outputFile.getPatch(),
               );
@@ -160,7 +162,8 @@ export default class CandidatePatchGenerator extends Module {
               outputFile = await File.fileOf(
                 path.join(dirName, extractedFileName),
                 patch.getSizeAfter() ?? 0,
-                patch.getCrcAfter() ?? '00000000',
+                { crc32: patch.getCrcAfter() },
+                ChecksumBitmask.NONE,
                 outputFile.getFileHeader(),
                 outputFile.getPatch(),
               );

src/modules/candidateWriter.ts

@@ -12,6 +12,7 @@ import Parent from '../types/dats/parent.js';
 import ArchiveEntry from '../types/files/archives/archiveEntry.js';
 import Zip from '../types/files/archives/zip.js';
 import File from '../types/files/file.js';
+import { ChecksumBitmask } from '../types/files/fileChecksums.js';
 import Options from '../types/options.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 import Module from './module.js';
@@ -213,7 +214,7 @@ export default class CandidateWriter extends Module {
 
     let archiveEntries: ArchiveEntry<Zip>[];
     try {
-      archiveEntries = await new Zip(zipFilePath).getArchiveEntries();
+      archiveEntries = await new Zip(zipFilePath).getArchiveEntries(ChecksumBitmask.CRC32);
     } catch (e) {
       return `failed to get archive contents: ${e}`;
     }

src/types/dats/rom.ts

@@ -3,6 +3,7 @@ import { Expose } from 'class-transformer';
 import Archive from '../files/archives/archive.js';
 import ArchiveEntry from '../files/archives/archiveEntry.js';
 import File from '../files/file.js';
+import { ChecksumProps } from '../files/fileChecksums.js';
 
 type ROMStatus = 'baddump' | 'nodump' | 'good';
 
@@ -83,7 +84,23 @@ export default class ROM implements ROMProps {
   }
 
   getCrc32(): string {
-    return this.crc ? this.crc.replace(/^0x/, '').padStart(8, '0') : '';
+    return (this.crc ?? '').toLowerCase().replace(/^0x/, '').padStart(8, '0');
+  }
+
+  getMd5(): string {
+    return (this.md5 ?? '').toLowerCase().replace(/^0x/, '').padStart(32, '0');
+  }
+
+  getSha1(): string {
+    return (this.sha1 ?? '').toLowerCase().replace(/^0x/, '').padStart(40, '0');
+  }
+
+  getChecksumProps(): ChecksumProps {
+    return {
+      crc32: this.getCrc32(),
+      md5: this.getMd5(),
+      sha1: this.getSha1(),
+    };
   }
 
   getStatus(): ROMStatus | undefined {
@@ -102,14 +119,14 @@ export default class ROM implements ROMProps {
    * Turn this {@link ROM} into a non-existent {@link File}.
    */
   async toFile(): Promise<File> {
-    return File.fileOf(this.getName(), this.getSize(), this.getCrc32());
+    return File.fileOf(this.getName(), this.getSize(), this.getChecksumProps());
   }
 
   /**
    * Turn this {@link ROM} into a non-existent {@link ArchiveEntry}, given a {@link Archive}.
    */
   async toArchiveEntry<A extends Archive>(archive: A): Promise<ArchiveEntry<A>> {
-    return ArchiveEntry.entryOf(archive, this.getName(), this.getSize(), this.getCrc32());
+    return ArchiveEntry.entryOf(archive, this.getName(), this.getSize(), this.getChecksumProps());
   }
 
   /**

src/types/files/archives/archive.ts

@@ -4,6 +4,7 @@ import { Readable } from 'node:stream';
 import Constants from '../../../constants.js';
 import fsPoly from '../../../polyfill/fsPoly.js';
 import File from '../file.js';
+import { ChecksumBitmask } from '../fileChecksums.js';
 import ArchiveEntry from './archiveEntry.js';
 
 export default abstract class Archive {
@@ -20,18 +21,19 @@ export default abstract class Archive {
    *  compute its size and CRC.
    */
   async asRawFile(): Promise<File> {
+    // TODO(cemmer): calculate MD5 and SHA1 for testing purposes?
     return File.fileOf(this.getFilePath());
   }
 
   async asRawFileWithoutCrc(): Promise<File> {
-    return File.fileOf(this.getFilePath(), undefined, '');
+    return File.fileOf(this.getFilePath(), undefined, undefined, ChecksumBitmask.NONE);
   }
 
   getFilePath(): string {
     return this.filePath;
   }
 
-  abstract getArchiveEntries(): Promise<ArchiveEntry<Archive>[]>;
+  abstract getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Archive>[]>;
 
   abstract extractEntryToFile(
     entryPath: string,

src/types/files/archives/archiveEntry.ts

@@ -5,6 +5,7 @@ import Constants from '../../../constants.js';
 import fsPoly from '../../../polyfill/fsPoly.js';
 import Patch from '../../patches/patch.js';
 import File, { FileProps } from '../file.js';
+import { ChecksumBitmask, ChecksumProps } from '../fileChecksums.js';
 import ROMHeader from '../romHeader.js';
 import Archive from './archive.js';
 
@@ -27,32 +28,60 @@ export default class ArchiveEntry<A extends Archive> extends File implements Arc
   static async entryOf<A extends Archive>(
     archive: A,
     entryPath: string,
-    size: number,
-    crc: string,
+    size?: number,
+    checksums?: ChecksumProps,
+    checksumBitmask: number = ChecksumBitmask.CRC32,
     fileHeader?: ROMHeader,
     patch?: Patch,
   ): Promise<ArchiveEntry<A>> {
+    let finalSize = size;
+    let finalCrcWithHeader = checksums?.crc32;
     let finalCrcWithoutHeader;
+    let finalMd5 = checksums?.md5;
+    let finalSha1 = checksums?.sha1;
     let finalSymlinkSource;
+
     if (await fsPoly.exists(archive.getFilePath())) {
-      if (await fsPoly.isSymlink(archive.getFilePath())) {
-        finalSymlinkSource = await fsPoly.readlink(archive.getFilePath());
+      // Calculate checksums
+      if ((!finalCrcWithHeader && (checksumBitmask & ChecksumBitmask.CRC32))
+        || (!finalMd5 && (checksumBitmask & ChecksumBitmask.MD5))
+        || (!finalSha1 && (checksumBitmask & ChecksumBitmask.SHA1))
+      ) {
+        const calculatedChecksums = await this.extractEntryToTempFile(
+          archive,
+          entryPath,
+          async (localFile) => this.calculateChecksums(localFile, checksumBitmask),
+        );
+        finalCrcWithHeader = calculatedChecksums.crc32 ?? finalCrcWithHeader;
+        finalMd5 = calculatedChecksums.md5 ?? finalMd5;
+        finalSha1 = calculatedChecksums.sha1 ?? finalSha1;
       }
       if (fileHeader) {
-        finalCrcWithoutHeader = finalCrcWithoutHeader ?? await this.extractEntryToTempFile(
+        finalCrcWithoutHeader = (await this.extractEntryToTempFile(
           archive,
           entryPath,
-          async (localFile) => this.calculateCrc32(localFile, fileHeader),
-        );
+          async (localFile) => this.calculateChecksums(
+            localFile,
+            ChecksumBitmask.CRC32,
+            fileHeader,
+          ),
+        )).crc32;
+      }
+
+      if (await fsPoly.isSymlink(archive.getFilePath())) {
+        finalSymlinkSource = await fsPoly.readlink(archive.getFilePath());
       }
     }
-    finalCrcWithoutHeader = finalCrcWithoutHeader ?? crc;
+    finalSize = finalSize ?? 0;
+    finalCrcWithoutHeader = finalCrcWithoutHeader ?? finalCrcWithHeader;
 
     return new ArchiveEntry<A>({
       filePath: archive.getFilePath(),
-      size,
-      crc32: crc,
+      size: finalSize,
+      crc32: finalCrcWithHeader,
       crc32WithoutHeader: finalCrcWithoutHeader,
+      md5: finalMd5,
+      sha1: finalSha1,
       symlinkSource: finalSymlinkSource,
       fileHeader,
       patch,
@@ -127,15 +156,11 @@ export default class ArchiveEntry<A extends Archive> extends File implements Arc
     });
   }
 
-  async withEntryPath(entryPath: string): Promise<ArchiveEntry<A>> {
-    return ArchiveEntry.entryOf(
-      this.getArchive(),
+  withEntryPath(entryPath: string): ArchiveEntry<A> {
+    return new ArchiveEntry({
+      ...this,
       entryPath,
-      this.getSize(),
-      this.getCrc32(),
-      this.getFileHeader(),
-      this.getPatch(),
-    );
+    });
   }
 
   async withFileHeader(fileHeader: ROMHeader): Promise<ArchiveEntry<A>> {
@@ -151,7 +176,8 @@ export default class ArchiveEntry<A extends Archive> extends File implements Arc
       this.getArchive(),
       this.getEntryPath(),
       this.getSize(),
-      this.getCrc32(),
+      this,
+      this.getChecksumBitmask(),
       fileHeader,
       undefined, // don't allow a patch
     );

src/types/files/archives/rar.ts

@@ -18,7 +18,7 @@ export default class Rar extends Archive {
   }
 
   @Memoize()
-  async getArchiveEntries(): Promise<ArchiveEntry<Rar>[]> {
+  async getArchiveEntries(checksumBitmask: number): Promise<ArchiveEntry<Rar>[]> {
     const rar = await unrar.createExtractorFromFile({
       filepath: this.getFilePath(),
     });
@@ -28,7 +28,9 @@ export default class Rar extends Archive {
         this,
         fileHeader.name,
         fileHeader.unpSize,
-        fileHeader.crc.toString(16),
+        { crc32: fileHeader.crc.toString(16) },
+        // If MD5 or SHA1 is desired, this file will need to be extracted to calculate
+        checksumBitmask,
       )));
   }
 

src/types/files/archives/sevenZip.ts

@@ -39,7 +39,7 @@ export default class SevenZip extends Archive {
   }
 
   @Memoize()
-  async getArchiveEntries(attempt = 1): Promise<ArchiveEntry<SevenZip>[]> {
+  async getArchiveEntries(checksumBitmask: number, attempt = 1): Promise<ArchiveEntry<SevenZip>[]> {
     /**
      * WARN(cemmer): {@link _7z.list} seems to have issues with any amount of real concurrency,
      *  it will return no files but also no error. Try to prevent that behavior.
@@ -71,7 +71,7 @@ export default class SevenZip extends Archive {
       await new Promise((resolve) => {
         setTimeout(resolve, Math.random() * (2 ** (attempt - 1) * 100));
       });
-      return this.getArchiveEntries(attempt + 1);
+      return this.getArchiveEntries(checksumBitmask, attempt + 1);
     }
 
     return Promise.all(filesIn7z
@@ -80,7 +80,9 @@ export default class SevenZip extends Archive {
         this,
         result.name,
         parseInt(result.size, 10),
-        result.crc,
+        { crc32: result.crc },
+        // If MD5 or SHA1 is desired, this file will need to be extracted to calculate
+        checksumBitmask,
       )));
   }