From be5261f6e2d9062925197d7e18fc63b4be69d324 Mon Sep 17 00:00:00 2001 From: Max Thomson Date: Thu, 6 Jun 2024 10:37:38 -0700 Subject: [PATCH 1/3] [CI] Implement BANNER API course database population --- functions/src/courses/Course.service.ts | 108 ++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/functions/src/courses/Course.service.ts b/functions/src/courses/Course.service.ts index b2413766..cb5b88a2 100644 --- a/functions/src/courses/Course.service.ts +++ b/functions/src/courses/Course.service.ts @@ -14,6 +14,114 @@ import { import { getSections } from '../sections/Section.service'; import { mapLimit } from 'async'; +export interface BannerApiResponse { + success: boolean; + totalCount: number; + data: Class[]; + pageOffset: number; + pageMaxSize: number; + sectionsFetchedCount: number; + pathMode: string; + searchResultsConfigs: null; + ztcEncodedImage: null; + allowHoldRegistration: null; +} + +export interface Class { + id: number; + term: string; + termDesc: string; + courseReferenceNumber: string; + partOfTerm: string; + courseNumber: string; + subject: string; + subjectDescription: string; + sequenceNumber: string; + campusDescription: "Main" | "Off Campus" | "Online" | "Victoria, BC"; + scheduleTypeDescription: "Gradable Lab" | "Lab" | "Lecture" | "Lecture Topic" | "Tutorial"; + courseTitle: string; + creditHours: number | null; + maximumEnrollment: number; + enrollment: number; + seatsAvailable: number; + waitCapacity: number; + waitCount: number; + waitAvailable: number; + crossList: null; + crossListCapacity: null; + crossListCount: null; + crossListAvailable: null; + creditHourHigh: number | null; + creditHourLow: number; + creditHourIndicator: "OR" | "TO" | null; + openSection: boolean; + linkIdentifier: null | string; + isSectionLinked: boolean; + subjectCourse: string; + faculty: Faculty[]; + meetingsFaculty: MeetingsFaculty[]; + reservedSeatSummary: null; + sectionAttributes: null; + instructionalMethod: "F2F" | "OL"; + instructionalMethodDescription: "Face-to-face" | "Fully Online"; +} + +export interface Faculty { + bannerId: string; + category: null; + class: string; + courseReferenceNumber: string; + displayName: string; + emailAddress: string; + primaryIndicator: boolean; + term: string; +} + +export interface MeetingsFaculty { + category: string; + class: string; + courseReferenceNumber: string; + faculty: any[]; + meetingTime: MeetingTime; + term: string; +} + +export interface MeetingTime { + beginTime: null | string; + building: null | string; + buildingDescription: null | string; + campus: "M" | null; + campusDescription: "Main" | "Off Campus" | "Online" | "Victoria, BC" | null; + category: string; + class: string; + courseReferenceNumber: string; + creditHourSession: number; + endDate: string; + endTime: null | string; + friday: boolean; + hoursWeek: number; + meetingScheduleType: "GLB" | "L01" | "LAB" | "LEC" | "TUT"; + meetingType: "CLAS"; + meetingTypeDescription: "Every Week", + monday: boolean; + room: null | string; + saturday: boolean; + startDate: string; + sunday: boolean; + term: string; + thursday: boolean; + tuesday: boolean; + wednesday: boolean; +} + +type Section = { + subject: string; + code: string; + title: string; + pid: string; + sections: ClassScheduleListing[]; +}; + export class CoursesService { /** * From 5b89f0c48207980cd32e171ff5380d2ac49517bf Mon Sep 17 00:00:00 2001 From: Max Thomson Date: Thu, 6 Jun 2024 11:37:21 -0700 Subject: [PATCH 2/3] Initial api call to banner search & subsquent logoff --- functions/scripts/populate-courses.ts | 13 ++++-- functions/src/courses/Course.service.ts | 53 +++++++++++++++---------- functions/tsconfig.json | 2 +- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/functions/scripts/populate-courses.ts b/functions/scripts/populate-courses.ts index 532568f2..d1251e25 100644 --- a/functions/scripts/populate-courses.ts +++ b/functions/scripts/populate-courses.ts @@ -11,16 +11,21 @@ if (process.env.FIRESTORE_EMULATOR_HOST) { admin.initializeApp({ credential: admin.credential.applicationDefault() }); } -if (process.argv.length != 3) throw Error('Term argument not found.'); +if (process.argv.length != 4) throw Error(`usage: ${process.argv[0]} ${process.argv[1]} [term] [cookie]`); -const term = process.argv[2]; +const term = process.argv[2].trim(); -if (!/20\d{2}0[1,5,9]/.test(term.trim())) +if (!/^20\d{2}0[1,5,9]$/.test(term)) throw Error('Invalid term argument format'); +const cookie = process.argv[3].trim(); + +if (!/^[0-9A-F]{32}$/.test(cookie)) + throw Error('Invalid cookie argument format'); + const main = async () => { console.log('Populating Firestore with data...'); - await CoursesService.populateCourses(term as Term); + await CoursesService.populateCourses(term as Term, cookie); }; main(); diff --git a/functions/src/courses/Course.service.ts b/functions/src/courses/Course.service.ts index cb5b88a2..dca64d0d 100644 --- a/functions/src/courses/Course.service.ts +++ b/functions/src/courses/Course.service.ts @@ -201,30 +201,40 @@ export class CoursesService { * NOTE: the assumption is this won't be run very often. * @param term */ - static async populateCourses(term: Term): Promise { - console.log('Fetching courses...'); - const courses = await CoursesService.getCourses(term); + static async populateCourses(term: Term, cookie: string): Promise { // get all sections for a given term and course console.log('Fetching sections...'); - const sections = await mapLimit< - Course, - { - subject: string; - code: string; - title: string; - pid: string; - sections: ClassScheduleListing[]; - }, - Error - >(courses, 50, async ({ subject, code, title, pid }) => ({ - // makes iterating over the data easier if we have the subject and code. - sections: await getSections(term, subject, code), - subject, - code, - title, - pid, - })); + const querySize = 2; // Set to arbitrarily large number (5 digits) to get all sections + const resp = await fetch(`https://banner.uvic.ca/StudentRegistrationSsb/ssb/searchResults/searchResults?txt_term=${term}&pageMaxSize=${querySize}&sortColumn=subjectDescription&sortDirection=asc`, { + headers: { + 'Accept': 'application/json', + 'Cookie': `JSESSIONID=${cookie}`, + 'Pragma': 'no-cache', + 'Cache-Control': 'no-cache' + } + }); + + await fetch("https://banner.uvic.ca/StudentRegistrationSsb/logoff", { + "credentials": "include", + headers: { + "Accept": "text/html", + 'Cookie': `JSESSIONID=${cookie}`, + 'Pragma': 'no-cache', + 'Cache-Control': 'no-cache', + } + }); + console.log(`Logout status: ${resp.status}`) + + const jsn = await resp.json() as BannerApiResponse + if (!jsn || jsn.success == false || !jsn.data || jsn.data.length == 0) + throw Error(`Banner API request failed: ${JSON.stringify(jsn)}`); + + + console.log(JSON.stringify(jsn)) + + /* + const sections: Section[] = {}; console.log( `Inserting ${sections.length} records as batch operation into Firestore...` @@ -281,6 +291,7 @@ export class CoursesService { } console.log(`Flushing remaining courses...`); await commit(); + */ } } diff --git a/functions/tsconfig.json b/functions/tsconfig.json index cdc155d1..2496e210 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -4,7 +4,7 @@ "module": "commonjs", "resolveJsonModule": true, "noImplicitReturns": true, - "noUnusedLocals": true, + // "noUnusedLocals": true, "outDir": "lib", "sourceMap": true, "strict": true, From 1825530235fcc484a53335b33e6e78cbf127b64b Mon Sep 17 00:00:00 2001 From: Max Thomson Date: Thu, 6 Jun 2024 11:53:07 -0700 Subject: [PATCH 3/3] Update CI workflow to take `cookie` as input --- .github/workflows/populate-courses.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/populate-courses.yaml b/.github/workflows/populate-courses.yaml index 342b70f4..c2a6cefe 100644 --- a/.github/workflows/populate-courses.yaml +++ b/.github/workflows/populate-courses.yaml @@ -8,6 +8,9 @@ on: term: description: Academic term to execute descript with. ie 202009, 202001 required: true + cookie: + description: JSESSIONID Cookie for the '/StudentRegistrationSsb' Path + required: true jobs: lint: name: Lint @@ -41,6 +44,7 @@ jobs: run: working-directory: ./functions steps: + - run: echo "::add-mask::${{ github.event.inputs.cookie }}" - uses: actions/checkout@v2 - name: Setup Node.js uses: actions/setup-node@v1 @@ -61,4 +65,4 @@ jobs: service_account_key: ${{ secrets.FIREBASE_SERVICE_ACCOUNT }} export_default_credentials: true - name: Execute Script - run: npm run db:populate ${{ github.event.inputs.term }} + run: npm run db:populate "${{ github.event.inputs.term }}" "${{ github.event.inputs.cookie }}"