Merge remote-tracking branch 'origin/develop' into feature/issue-871_…

…tabulate-by-csv-updates
BrightSpots · Sep 4, 2024 · eb4c928 · eb4c928
2 parents a632d92 + 85b3455
commit eb4c928
Show file tree

Hide file tree

Showing 134 changed files with 3,017 additions and 2,703 deletions.
diff --git a/.github/actions/sha-of-zip.bat b/.github/actions/sha-of-zip.bat
@@ -58,8 +58,9 @@ powershell -Command "& {[IO.File]::WriteAllText(\"%HASHFILE_SORTED%\", $([IO.Fil
 :: echo the final hash
 C:\Windows\System32\certutil.exe -hashfile %HASHFILE_SORTED% SHA%SHA_A% | findstr /v ":"
 
-:: For debugging, enable printing the file-by-file hash
-:: echo File-by-file hash
-:: type "%HASHFILE_SORTED%"
+:: For easier debugging, print the file-by-file hash
+echo ##########
+echo This checksum was created by a SHA-%SHA_A% of the following file, after removing the filenames and sorting by their SHAs:
+sort "%HASHFILE_SORTED%"
 
 endlocal
diff --git a/.github/actions/sha-of-zip.sh b/.github/actions/sha-of-zip.sh
@@ -15,7 +15,9 @@ parentPath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 # Make a temporary directory to extract zip, and a temporary file to hold SHAs
 tempDirectory=$(mktemp -d)
 tempAllChecksumsFile=$(mktemp)
+tempChecksumDebugInfoFile=$(mktemp)
 touch $tempAllChecksumsFile
+touch $tempChecksumDebugInfoFile
 
 # Extract the zip
 unzip -q $zipFilepath -d $tempDirectory 2>/dev/null
@@ -25,7 +27,13 @@ cd $tempDirectory
 for filename in $(find * -type f | sort); do
   checksum=$($parentPath/../workflows/sha.sh $filename $os $sha_a)
   echo $checksum >> $tempAllChecksumsFile
+  echo $filename = $checksum >> $tempChecksumDebugInfoFile
 done
 
 # Echo the checksum of the checksums
 echo $($parentPath/../workflows/sha.sh $tempAllChecksumsFile $os $sha_a)
+
+# For easier debugging, print the file-by-file hash
+echo "##########"
+echo "This checksum was created by a SHA-$sha_a of the following file, after removing the filenames and sorting by their SHAs:"
+cat $tempChecksumDebugInfoFile | sort
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -104,6 +104,11 @@ jobs:
           input: "~/.gradle/caches"
           zipFilename: ${{steps.cachefn.outputs.FILEPATH}}
 
+      - name: "Generate SHA512 for executable"
+        shell: bash
+        run: |
+          ./.github/workflows/sha.sh ${{ steps.exefn.outputs.FILEPATH }} ${{ runner.os }} 512 > ${{steps.exefn.outputs.FILEPATH}}.sha512
+
       - name: "Generate SHA512 for plugins cache"
         shell: bash
         run: |
@@ -166,10 +171,11 @@ jobs:
           echo "Attach staple"
           xcrun stapler staple ${{ steps.exefn.outputs.FILEPATH }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Package
           if-no-files-found: error
+          overwrite: true
           path: |
             ${{ github.workspace }}/${{    steps.zipfn.outputs.FILEPATH    }}
             ${{ github.workspace }}/${{    steps.zipfn.outputs.FILEPATH    }}.sha512
@@ -180,14 +186,33 @@ jobs:
             ${{ github.workspace }}/${{   steps.cachefn.outputs.FILEPATH   }}.sha512
             ${{ github.workspace }}/${{ steps.checksumsfn.outputs.FILEPATH }}
             ${{ github.workspace }}/${{ steps.checksumsfn.outputs.FILEPATH }}.sha512
-          retention-days: 90
+
+      - uses: dev-drprasad/[email protected]
+        # Note: to update the date of the release, it must be deleted and then recreated
+        if: github.event_name == 'release' || github.event_name == 'schedule'
+        with:
+          tag_name: build-${{ github.ref_name }}
+          delete_release: true
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Upload binaries to release"
         uses: svenstaro/upload-release-action@v2
         if: github.event_name == 'release' || github.event_name == 'schedule'
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: build/${{ steps.basefn.outputs.FILEPATH }}*
-          tag: ${{ github.ref_name }}
+          tag: build-${{ github.ref_name }}
+          overwrite: true
+          file_glob: true
+          make_latest: false
+
+      - name: "Upload cache to release"
+        uses: svenstaro/upload-release-action@v2
+        if: github.event_name == 'release' || github.event_name == 'schedule'
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: cache/*
+          tag: build-${{ github.ref_name }}
           overwrite: true
           file_glob: true
+          make_latest: false
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,6 +1,6 @@
 name: "Build, Lint, and Test"
 
-on: [push]
+on: [pull_request]
 
 jobs:
   build:

diff --git a/README.md b/README.md
@@ -32,13 +32,15 @@ On Linux, you may install the .deb file, then run `/opt/rcv/bin/RCTab` to launch
 
 #### Method 2 (Less Easy): Compile and Run Using Gradle
 
-1. Install [JDK 20 or higher](https://jdk.java.net/), and make sure your Java path is picking it up properly by
+1. Install [JDK 20](https://www.oracle.com/java/technologies/javase/jdk20-archive-downloads.html), and make sure your Java path is picking it up properly by
    verifying that the following command returns the expected version:
 
     `$ java -version`
 
     If the expected version isn't returned, you'll need to follow the instructions [here](https://www.java.com/en/download/help/path.xml) on how to set your Java path.
 
+    If you are using Linux or MacOS and need to regularly switch between Java versions, consider installing [jEnv](https://www.jenv.be/). For a list of the Java versions installed on your machine, run `/usr/libexec/java_home -V` on MacOS or `update-alternatives --config java` on Linux.
+
 2. Download the [zip of the source code from GitHub](https://github.com/BrightSpots/rcv/archive/master.zip) and unzip it, or install git and use the following command at the terminal / command prompt to clone a local copy on your machine:
 
     `$ git clone https://github.com/BrightSpots/rcv.git`

diff --git a/src/main/java/network/brightspots/rcv/BaseCvrReader.java b/src/main/java/network/brightspots/rcv/BaseCvrReader.java
@@ -119,6 +119,15 @@ Map<String, Integer> gatherUnknownCandidates(
     return unrecognizedCandidateCounts;
   }
 
+  boolean usesLastAllowedRanking(List<Pair<Integer, String>> rankings, String contestId) {
+    if (rankings.isEmpty()) {
+      return false;
+    }
+
+    int lastRanking = rankings.get(rankings.size() - 1).getKey();
+    return !isRankingAllowed(lastRanking + 1, contestId);
+  }
+
   // Human-readable name for output logs
   public abstract String readerName();
 }
diff --git a/src/main/java/network/brightspots/rcv/CastVoteRecord.java b/src/main/java/network/brightspots/rcv/CastVoteRecord.java
@@ -36,6 +36,8 @@ class CastVoteRecord {
   private final String precinct;
   // which precinct portion this ballot came from
   private final String precinctPortion;
+  // is the last-used ranking the last-allowed ranking in the CVR?
+  private final boolean usesLastAllowedRanking;
   // records winners to whom some fraction of this vote has been allocated
   private final Map<String, BigDecimal> winnerToFractionalValue = new HashMap<>();
   // If CVR CDF output is enabled, we store the necessary info here: for each round, the list of
@@ -65,8 +67,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String precinctPortion,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion, rankings);
+    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion,
+        usesLastAllowedRanking, rankings);
   }
 
   CastVoteRecord(
@@ -77,6 +81,7 @@ class CastVoteRecord {
           String computedId,
           String precinct,
           String precinctPortion,
+          boolean usesLastAllowedRanking,
           List<Pair<Integer, String>> rankings) {
     this.contestId = contestId;
     this.tabulatorId = tabulatorId;
@@ -85,6 +90,7 @@ class CastVoteRecord {
     this.computedId = computedId;
     this.precinct = precinct;
     this.precinctPortion = precinctPortion;
+    this.usesLastAllowedRanking = usesLastAllowedRanking;
     this.candidateRankings = new CandidateRankingsList(rankings);
   }
 
@@ -93,8 +99,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String batchId,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
+    this(null, null, batchId, suppliedId, computedId, precinct, null,
+        usesLastAllowedRanking, rankings);
   }
 
   String getContestId() {
@@ -116,6 +124,10 @@ String getPrecinctPortion() {
     return precinctPortion;
   }
 
+  boolean doesUseLastAllowedRanking() {
+    return usesLastAllowedRanking;
+  }
+
   String getId() {
     return suppliedId != null ? suppliedId : computedId;
   }
@@ -224,11 +236,11 @@ Map<String, BigDecimal> getWinnerToFractionalValue() {
   // it is active or not.
   enum StatusForRound {
     ACTIVE,
-    INACTIVE_BY_UNDERVOTE,
-    INACTIVE_BY_OVERVOTE,
-    INACTIVE_BY_SKIPPED_RANKING,
-    INACTIVE_BY_REPEATED_RANKING,
-    INACTIVE_BY_EXHAUSTED_CHOICES
+    DID_NOT_RANK_ANY_CANDIDATES,
+    EXHAUSTED_CHOICE,
+    INVALIDATED_BY_OVERVOTE,
+    INVALIDATED_BY_SKIPPED_RANKING,
+    INVALIDATED_BY_REPEATED_RANKING,
   }
 
   enum VoteOutcomeType {

diff --git a/src/main/java/network/brightspots/rcv/ClearBallotCvrReader.java b/src/main/java/network/brightspots/rcv/ClearBallotCvrReader.java
@@ -110,6 +110,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
                 cvrData[CvrColumnField.BallotID.ordinal()],
                 cvrData[CvrColumnField.PrecinctID.ordinal()],
                 null,
+                usesLastAllowedRanking(rankings, null),
                 rankings);
 
         castVoteRecords.add(castVoteRecord);

diff --git a/src/main/java/network/brightspots/rcv/CommonDataFormatReader.java b/src/main/java/network/brightspots/rcv/CommonDataFormatReader.java
@@ -316,7 +316,12 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO
         String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
         // create the new CastVoteRecord
         CastVoteRecord newRecord = new CastVoteRecord(
-            computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
+            computedCastVoteRecordId,
+            cvr.UniqueId,
+            precinctId,
+            cvr.BatchSequenceId,
+            usesLastAllowedRanking(rankings, null),
+            rankings);
         castVoteRecords.add(newRecord);
 
         // provide some user feedback on the CVR count
@@ -527,8 +532,13 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
       String batchId = (String) cvr.get("BatchSequenceId");
       String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
       // create the new CastVoteRecord
-      CastVoteRecord newRecord =
-          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
+      CastVoteRecord newRecord = new CastVoteRecord(
+          computedCastVoteRecordId,
+          ballotId,
+          precinctId,
+          batchId,
+          usesLastAllowedRanking(rankings, null),
+          rankings);
       castVoteRecords.add(newRecord);
       // provide some user feedback on the CVR count
       if (castVoteRecords.size() % 50000 == 0) {

diff --git a/src/main/java/network/brightspots/rcv/CsvCvrReader.java b/src/main/java/network/brightspots/rcv/CsvCvrReader.java
@@ -99,9 +99,13 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         }
 
         // create the new CastVoteRecord
-        CastVoteRecord newCvr =
-            new CastVoteRecord(
-                Integer.toString(index), "no supplied ID", "no precinct", "no batch ID", rankings);
+        CastVoteRecord newCvr = new CastVoteRecord(
+            Integer.toString(index),
+            "no supplied ID",
+            "no precinct",
+            "no batch ID",
+            usesLastAllowedRanking(rankings, null),
+            rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

diff --git a/src/main/java/network/brightspots/rcv/DominionCvrReader.java b/src/main/java/network/brightspots/rcv/DominionCvrReader.java
@@ -357,9 +357,16 @@ private int parseCvrFile(
             rankings.add(ranking);
           }
           // create the new cvr
-          CastVoteRecord newCvr =
-              new CastVoteRecord(contestId, tabulatorId, batchId, suppliedId,
-                      computedId, precinct, precinctPortion, rankings);
+          CastVoteRecord newCvr = new CastVoteRecord(
+              contestId,
+              tabulatorId,
+              batchId,
+              suppliedId,
+              computedId,
+              precinct,
+              precinctPortion,
+              usesLastAllowedRanking(rankings, contestId),
+              rankings);
           castVoteRecords.add(newCvr);
         }
       }