From 8f92e078bfe1c7099cecbbd68477939de7197e0a Mon Sep 17 00:00:00 2001 From: Ty Date: Mon, 18 Mar 2024 23:31:01 -0600 Subject: [PATCH] Initial commit --- .gitignore | 2 + .vscode/settings.json | 3 + cli.ts | 221 ++++++++++++++++++++++++++++++++++++++++++ types.ts | 122 +++++++++++++++++++++++ 4 files changed, 348 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 cli.ts create mode 100644 types.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43e3639 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +auth.txt +out.csv \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..cbac569 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "deno.enable": true +} diff --git a/cli.ts b/cli.ts new file mode 100644 index 0000000..7f4b6fc --- /dev/null +++ b/cli.ts @@ -0,0 +1,221 @@ +import { + DOMParser, +} from "https://deno.land/x/deno_dom@v0.1.45/deno-dom-wasm.ts"; +import { writeCSV } from "https://deno.land/x/csv@v0.9.2/mod.ts"; + +import { + NSDAMemberLookup, + NSDAMemberPointEntry, + NSDAMemberSearchResult, + TabroomEntry, +} from "./types.ts"; + +// Load auth +const nsdaAuth = await Deno.readFile("./auth.txt").then((a) => + new TextDecoder().decode(a) +); + +// Fetch and parse tabroom USX entries +const tabroomHtml = await fetch( + "https://www.tabroom.com/index/tourn/fields.mhtml?tourn_id=30061&event_id=279207", +).then((r) => r.text()); + +const dom = new DOMParser().parseFromString( + tabroomHtml, + "text/html", +)!; + +const entries: TabroomEntry[] = []; + +for (const row of dom.querySelectorAll("tr")) { + const [institution, location, entry, code] = [...row.childNodes] + .filter((n) => n.nodeName === "TD") + .map((n) => n.textContent.trim()); + // Skip invalid entries + if (!institution || !location || !entry || !code) continue; + + entries.push({ + institution, + location, + entry, + code, + }); +} + +// Lookup on NSDA +const nsdaSearchResults = await Promise + .all( + entries.map(async (e) => { + let nsdaLookup: NSDAMemberSearchResult[] = await fetch( + `https://api.speechanddebate.org/v2/search?${ + new URLSearchParams({ + q: e.entry, + type: "members", + }).toString() + }`, + { + headers: { + authorization: nsdaAuth, + }, + }, + ).then((r) => r.json()); + + // Do our best to filter the results down to one + // Start by filtering anything but HS competitors and status === 1 + // (status 2 appeared on a duplicate, so I assume that marks inactive entries?) + nsdaLookup = nsdaLookup.filter((ee) => + ee.realm === "hs" && ee.status === 1 + ); + // Then, if that doesn't work, filter by state + if (nsdaLookup.length > 1) { + nsdaLookup = nsdaLookup.filter((ee) => + ee.school_state.toLowerCase() === + e.location.replace("/US", "").toLowerCase() + ); + } + // Finally, if none of the past filters worked, attempt a school filter + let schoolFilterUsed = false; + if (nsdaLookup.length > 1) { + schoolFilterUsed = true; + + nsdaLookup = nsdaLookup.filter((ee) => + ee.school_name.toLowerCase().includes(e.institution.toLowerCase()) + ); + } + + if (nsdaLookup.length < 1) { + console.error( + `No results found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${ + e.location.replace("/US", "") + }.`, + ); + if (schoolFilterUsed) console.error("SCHOOL-BASED FILTER WAS USED!"); + return null; + } else if (nsdaLookup.length > 1) { + console.error( + `Multiple entries found for entry "${e.entry}" (with code "${e.code}"), from ${e.institution} in ${ + e.location.replace("/US", "") + }!`, + ); + console.error("---------------------------------------------"); + console.error(nsdaLookup); + console.error("---------------------------------------------"); + return null; + } else { + return { tabroom: e, nsdaSearchResult: nsdaLookup[0] }; + } + }), + ).then((p) => + p.filter((e) => e !== null) as { + tabroom: TabroomEntry; + nsdaSearchResult: NSDAMemberSearchResult; + }[] + ); + +// Convert NSDA searches to true NSDA member objects and point entries to get actual information +const nsdaLookups = await Promise.all( + nsdaSearchResults.map(async ({ tabroom, nsdaSearchResult }) => ({ + tabroom, + nsdaSearchResult, + nsdaMemberLookup: await fetch( + "https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id, + { headers: { authorization: nsdaAuth } }, + ).then((r) => r.json()) as NSDAMemberLookup, + nsdaPointsLookup: await fetch( + "https://api.speechanddebate.org/v2/members/" + nsdaSearchResult.id + + "/points", + { headers: { authorization: nsdaAuth } }, + ).then((r) => r.json()) as NSDAMemberPointEntry[], + })), +); + +// Analyze and display aggregate fetched information +console.log( + `Represented states: +${ + Object.entries(nsdaLookups.reduce( + (acc, l) => ({ + ...acc, + [l.nsdaSearchResult.school_state]: + (acc[l.nsdaSearchResult.school_state] ?? 0) + 1, + }), + {} as Record, + )).sort((a, b) => b[1] - a[1]).map(([state, n]) => "- " + state + ": " + n) + .join("\n") + } +Total entries: ${nsdaLookups.length} +Total schools: ${ + [...new Set(nsdaLookups.map((l) => l.nsdaSearchResult.school_name))].length + }`, +); + +// Collect information into a spreadsheet format +const sheet = [ + [ + "Last points entry", + "First name", + "Middle name", + "Last name", + "State", + "School", + "Extemp Points", + "Year Extemp Points", + "Extemp tournaments", + "Year Extemp Tournaments", + "Average Placing", + "Year Average Placing", + ], + ...nsdaLookups.map((l) => [ + l.nsdaMemberLookup.last_points_entry, + l.nsdaMemberLookup.first, + l.nsdaMemberLookup.middle ?? "", + l.nsdaMemberLookup.last, + l.nsdaSearchResult.school_state, + l.nsdaSearchResult.school_name, + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 /* Extemporaneous speech event ID */ + ).reduce((acc, e) => acc + e.points, 0).toString(), + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 && /* Extemporaneous speech event ID */ + new Date(e.start).getTime() > new Date(2023, 6, 0).getTime() + ).reduce((acc, e) => acc + e.points, 0).toString(), + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 /* Extemporaneous speech event ID */ + ).reduce( + (acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])], + [] as number[], + ).length.toString(), + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 && /* Extemporaneous speech event ID */ + new Date(e.start).getTime() > new Date(2023, 6, 0).getTime() + ).reduce( + (acc, e) => [...acc, ...(acc.includes(e.tourn_id) ? [] : [e.tourn_id])], + [] as number[], + ).length.toString(), + (Math.round( + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 /* Extemporaneous speech event ID */ + ).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks) + .reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000, + ) / 1000).toString(), + (Math.round( + l.nsdaPointsLookup.filter((e) => + e.category_id === 202 && /* Extemporaneous speech event ID */ + new Date(e.start).getTime() > new Date(2023, 6, 0).getTime() + ).flatMap((e) => (JSON.parse(e.result) as { ranks: number[] }).ranks) + .reduce((acc, v, _, a) => (acc + v / a.length), 0) * 1000, + ) / 1000).toString(), + ]), +]; + +await writeCSV( + await Deno.open("./out.csv", { + write: true, + create: true, + truncate: true, + }), + sheet, + { + columnSeparator: "\t", + }, +); diff --git a/types.ts b/types.ts new file mode 100644 index 0000000..6533709 --- /dev/null +++ b/types.ts @@ -0,0 +1,122 @@ +export interface NSDAMemberSearchResult { + id: number; + first: string; + last: string; + role: string; + school_id: number; + status: number; + realm: string; + school_name: string; + school_state: string; + type: string; + name: string; + description: string; +} + +export interface TabroomEntry { + institution: string; + location: string; + entry: string; + code: string; +} + +export interface NSDAMemberLookup { + person_id: number; + first: string; + middle: string | null; + last: string; + disabled: number; + created_at: string; + points: number; + points_service: number; + points_service_this_year: number; + points_this_year: number; + points_last_year: number; + last_points_entry: string; + citation_points: number; + degree_id: number; + degree_name: string; + to_next_degree: number; + diamonds: number | null; + "3_diamond": boolean; + hof: boolean; + aaa: boolean; + active: boolean; + paid: boolean; + paid_latest: string; + school_paid: boolean; + active_student: boolean; + degrees: NSDAMemberDegree[]; + honors: NSDAMemberHonor[]; + citations: unknown[]; + districts_eligible: boolean; +} + +export interface NSDAMemberDegree { + member_honor_id: number; + person_id: number; + honor_id: number; + description: string; + type: string; + school_id: number; + school_name: string; + district_id: number | null; + district_name: string | null; + state: string | null; + points: number; + start: string; + end: unknown; + status: string; + recognized: unknown; + note: string | null; + created_at: string; + created_by_id: unknown; +} + +export interface NSDAMemberHonor { + member_honor_id: number; + person_id: number; + honor_id: number; + description: string; + type: string; + school_id: number; + school_name: string; + district_id: number | null; + district_name: string | null; + state: string | null; + points: number | null; + start: string; + end: unknown; + status: string; + recognized: unknown; + note: string | null; + created_at: string; + created_by_id: unknown; +} + +export interface NSDAMemberPointEntry { + id: number; + student_id: number; + student_name: string; + coach_id: number; + coach_name: string; + school_id: number; + category_id: number; + points: number; + result: string; + description: string | null; + autopost: boolean; + districts: boolean; + nationals: boolean; + tourn_id: number; + tourn_name: string; + location: string; + state: string; + start: string; + end: string; + source: string; + status: number; + created_at: string; + created_by_id: number | null; + created_by: string; +}