[FIX] Implement BANNER API course database population

.github/workflows/populate-courses.yaml

@@ -8,6 +8,9 @@ on:
       term:
         description: Academic term to execute descript with. ie 202009, 202001
         required: true
+      cookie:
+        description: JSESSIONID Cookie for the '/StudentRegistrationSsb' Path
+        required: true
 jobs:
   lint:
     name: Lint
@@ -41,6 +44,7 @@ jobs:
       run:
         working-directory: ./functions
     steps:
+      - run: echo "::add-mask::${{ github.event.inputs.cookie }}"
       - uses: actions/checkout@v2
       - name: Setup Node.js
         uses: actions/setup-node@v1
@@ -61,4 +65,4 @@ jobs:
           service_account_key: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }}
           export_default_credentials: true
       - name: Execute Script
-        run: npm run db:populate ${{ github.event.inputs.term }}
+        run: npm run db:populate "${{ github.event.inputs.term }}" "${{ github.event.inputs.cookie }}"

functions/scripts/populate-courses.ts

@@ -11,16 +11,21 @@
   admin.initializeApp({ credential: admin.credential.applicationDefault() });
 }
 
-if (process.argv.length != 3) throw Error('Term argument not found.');
+if (process.argv.length != 4) throw Error(`usage: ${process.argv[0]} ${process.argv[1]} [term] [cookie]`);
 
-const term = process.argv[2];
+const term = process.argv[2].trim();
 
-if (!/20\d{2}0[1,5,9]/.test(term.trim()))
+if (!/^20\d{2}0[1,5,9]$/.test(term))
   throw Error('Invalid term argument format');
 
+const cookie = process.argv[3].trim();
+
+if (!/^[0-9A-F]{32}$/.test(cookie))
+  throw Error('Invalid cookie argument format');
+
 const main = async () => {
   console.log('Populating Firestore with data...');
-  await CoursesService.populateCourses(term as Term);
+  await CoursesService.populateCourses(term as Term, cookie);
 };
 
 main();

functions/src/courses/Course.service.ts

@@ -5,15 +5,123 @@
 } from '@vikelabs/uvic-course-scraper/dist/index';
 import { subjectCodeExtractor } from '../shared/subjectCodeExtractor';
 import { Term } from '../constants';
 import { query, where, batch, ref, get, update, set } from 'typesaurus';
 import {
   CourseDoc,
   CoursesCollection,
   SectionsSubstore,
 } from '../db/collections';
 import { getSections } from '../sections/Section.service';
 import { mapLimit } from 'async';
 
+export interface BannerApiResponse {
+  success: boolean;
+  totalCount: number;
+  data: Class[];
+  pageOffset: number;
+  pageMaxSize: number;
+  sectionsFetchedCount: number;
+  pathMode: string;
+  searchResultsConfigs: null;
+  ztcEncodedImage: null;
+  allowHoldRegistration: null;
+}
+
+export interface Class {
+  id: number;
+  term: string;
+  termDesc: string;
+  courseReferenceNumber: string;
+  partOfTerm: string;
+  courseNumber: string;
+  subject: string;
+  subjectDescription: string;
+  sequenceNumber: string;
+  campusDescription: "Main" | "Off Campus" | "Online" | "Victoria, BC";
+  scheduleTypeDescription: "Gradable Lab" | "Lab" | "Lecture" | "Lecture Topic" | "Tutorial";
+  courseTitle: string;
+  creditHours: number | null;
+  maximumEnrollment: number;
+  enrollment: number;
+  seatsAvailable: number;
+  waitCapacity: number;
+  waitCount: number;
+  waitAvailable: number;
+  crossList: null;
+  crossListCapacity: null;
+  crossListCount: null;
+  crossListAvailable: null;
+  creditHourHigh: number | null;
+  creditHourLow: number;
+  creditHourIndicator: "OR" | "TO" | null;
+  openSection: boolean;
+  linkIdentifier: null | string;
+  isSectionLinked: boolean;
+  subjectCourse: string;
+  faculty: Faculty[];
+  meetingsFaculty: MeetingsFaculty[];
+  reservedSeatSummary: null;
+  sectionAttributes: null;
+  instructionalMethod: "F2F" | "OL";
+  instructionalMethodDescription: "Face-to-face" | "Fully Online";
+}
+
+export interface Faculty {
+  bannerId: string;
+  category: null;
+  class: string;
+  courseReferenceNumber: string;
+  displayName: string;
+  emailAddress: string;
+  primaryIndicator: boolean;
+  term: string;
+}
+
+export interface MeetingsFaculty {
+  category: string;
+  class: string;
+  courseReferenceNumber: string;
+  faculty: any[];
+  meetingTime: MeetingTime;
+  term: string;
+}
+
+export interface MeetingTime {
+  beginTime: null | string;
+  building: null | string;
+  buildingDescription: null | string;
+  campus: "M" | null;
+  campusDescription: "Main" | "Off Campus" | "Online" | "Victoria, BC" | null;
+  category: string;
+  class: string;
+  courseReferenceNumber: string;
+  creditHourSession: number;
+  endDate: string;
+  endTime: null | string;
+  friday: boolean;
+  hoursWeek: number;
+  meetingScheduleType: "GLB" | "L01" | "LAB" | "LEC" | "TUT";
+  meetingType: "CLAS";
+  meetingTypeDescription: "Every Week",
+  monday: boolean;
+  room: null | string;
+  saturday: boolean;
+  startDate: string;
+  sunday: boolean;
+  term: string;
+  thursday: boolean;
+  tuesday: boolean;
+  wednesday: boolean;
+}
+
+type Section = {
+  subject: string;
+  code: string;
+  title: string;
+  pid: string;
+  sections: ClassScheduleListing[];
+};
+
 export class CoursesService {
   /**
    *
@@ -93,30 +201,40 @@
    * NOTE: the assumption is this won't be run very often.
    * @param term
    */
-  static async populateCourses(term: Term): Promise<void> {
-    console.log('Fetching courses...');
-    const courses = await CoursesService.getCourses(term);
+  static async populateCourses(term: Term, cookie: string): Promise<void> {
     // get all sections for a given term and course
     console.log('Fetching sections...');
 
-    const sections = await mapLimit<
-      Course,
-      {
-        subject: string;
-        code: string;
-        title: string;
-        pid: string;
-        sections: ClassScheduleListing[];
-      },
-      Error
-    >(courses, 50, async ({ subject, code, title, pid }) => ({
-      // makes iterating over the data easier if we have the subject and code.
-      sections: await getSections(term, subject, code),
-      subject,
-      code,
-      title,
-      pid,
-    }));
+    const querySize = 2; // Set to arbitrarily large number (5 digits) to get all sections
+    const resp = await fetch(`https://banner.uvic.ca/StudentRegistrationSsb/ssb/searchResults/searchResults?txt_term=${term}&pageMaxSize=${querySize}&sortColumn=subjectDescription&sortDirection=asc`, {
+      headers: {
+        'Accept': 'application/json',
+        'Cookie': `JSESSIONID=${cookie}`,
+        'Pragma': 'no-cache',
+        'Cache-Control': 'no-cache'
+      }
+    });
+
+    await fetch("https://banner.uvic.ca/StudentRegistrationSsb/logoff", {
+      "credentials": "include",
+      headers: {
+        "Accept": "text/html",
+        'Cookie': `JSESSIONID=${cookie}`,
+        'Pragma': 'no-cache',
+        'Cache-Control': 'no-cache',
+      }
+    });
+    console.log(`Logout status: ${resp.status}`)
+
+    const jsn = await resp.json() as BannerApiResponse
+    if (!jsn || jsn.success == false || !jsn.data || jsn.data.length == 0)
+      throw Error(`Banner API request failed: ${JSON.stringify(jsn)}`);
+
+
+    console.log(JSON.stringify(jsn))
+
+    /*
+    const sections: Section[] = {};
 
     console.log(
       `Inserting ${sections.length} records as batch operation into Firestore...`
@@ -173,6 +291,7 @@
     }
     console.log(`Flushing remaining courses...`);
     await commit();
+    */
   }
 }
 

functions/tsconfig.json

@@ -4,7 +4,7 @@
     "module": "commonjs",
     "resolveJsonModule": true,
     "noImplicitReturns": true,
-    "noUnusedLocals": true,
+    // "noUnusedLocals": true,
     "outDir": "lib",
     "sourceMap": true,
     "strict": true,