Take GitHub org config from yaml files

.env.example

@@ -2,33 +2,15 @@
 # Follow the instructions in README.md as how to add your personal
 # tokens and secrets in this file
 
-# GitHub App ID
-GH_APP_IDENTIFIER=''
-# Github App Webhook Secret to verify requests come from GitHub
-GH_WEBHOOK_SECRET=''
-# GitHub App Secret Key
-# NOTE: this *must* be on a single line, otherwise it will break
-# reading the secret key.
-GH_APP_SECRET_KEY='-----BEGIN RSA PRIVATE KEY----------END RSA PRIVATE KEY-----'
-# Personal (ie, for your own GitHub account) user access token with the `admin:org`,
-# `admin:org_hook`, `admin: repo_hook`, `project`, `public_repo`, `repo:status`, and
-# `repo_deployment` permissions enabled.
+# GitHub
 GH_USER_TOKEN=''
+GH_WEBHOOK_SECRET=''
+GH_APP_PRIVATE_KEY='-----BEGIN RSA PRIVATE KEY----------END RSA PRIVATE KEY-----'
+PERSONAL_TEST_REPO='testing-eng-pipes'
 
-# Slack signing secret to verify requests come from Slack
-SLACK_SIGNING_SECRET=
-# Slack Bot user access token -> "Bot User OAuth Token"
-SLACK_BOT_USER_ACCESS_TOKEN=
-
-# The name of the personal repository in your personal organization being used for development.
-PERSONAL_TEST_REPO='eng-pipes-dev'
-
-# Application-specific configuration
-# Config for the "GitHub Issues Someone Else Cares About" project board
-ISSUES_PROJECT_NODE_ID=''
-STATUS_FIELD_ID=''
-PRODUCT_AREA_FIELD_ID=''
-RESPONSE_DUE_DATE_FIELD_ID=''
+# Slack
+SLACK_SIGNING_SECRET=''
+SLACK_BOT_USER_ACCESS_TOKEN=''
 
 # Silence some GCP noise
 DRY_RUN=true

.env.test

@@ -4,17 +4,17 @@ GH_USER_TOKEN="ghp_BLAHBLAHBLAH"
 GH_APP_IDENTIFIER="1234"
 GH_WEBHOOK_SECRET="webhooksecret"
 GH_APP_SECRET_KEY="top \nsecret\n key"
+GETSENTRY_ORG="Enterprise"
 
 # Slack
 SLACK_SIGNING_SECRET="slacksigningsecret"
 SLACK_BOT_USER_ACCESS_TOKEN="slackbotaccesstoken"
 SLACK_BOT_APP_ID="5678"
 
 # Other
-OWNER="getsentry"
-SENTRY_REPO="sentry"
-GETSENTRY_REPO="getsentry"
 GETSENTRY_ORG="getsentry"
+SENTRY_REPO_SLUG="sentry"
+GETSENTRY_REPO_SLUG="getsentry"
 GOCD_SENTRYIO_FE_PIPELINE_NAME="getsentry-frontend"
 GOCD_SENTRYIO_BE_PIPELINE_NAME="getsentry-backend"
 GOCD_ORIGIN="https://deploy.getsentry.net"

.gitignore

@@ -19,3 +19,5 @@ lib
 !.yarn/plugins
 !.yarn/sdks
 !.yarn/versions
+
+github-orgs.local.yml

README.md

@@ -53,8 +53,7 @@ You can grab GitHub secrets in their respective configuration pages: [GitHub App
 
 You will also need to set up some of these environment variables if you want to test this locally, e.g. using `direnv` or something similar
 
-- `GH_APP_IDENTIFIER` - GitHub App identifier
-- `GH_APP_SECRET_KEY` - GitHub App private key
+- `GH_APP_PRIVATE_KEY` - GitHub App private key for your test app. It needs to all be on one line, but it can include literal '\n' which will be converted to newlines.
 - `GH_WEBHOOK_SECRET` - GitHub webhook secret to confirm that webhooks come from GitHub
 - `SENTRY_WEBPACK_WEBHOOK_SECRET` - Webhook secret that needs to match secret from CI. Records webpack asset sizes.
 - `SLACK_SIGNING_SECRET` - Slack webhook secret to verify payload
@@ -91,9 +90,7 @@ You'll also need to create a private key for the service account (it should down
 
 ## Configuring a test environment
 
-NOTE: These steps will cover more aspects over time. For now it focuses on testing Slack/Github changes.
-
-1. Set up [Ngrok](https://ngrok.io/) to redirect calls to your localhost.
+1. Set up [Ngrok](https://ngrok.io/) to redirect calls to your localhost ([smee.io](https://smee.io/) also works).
 
     - If you haven't already, sign up for an ngrok account and grab the auth token from [the setup page](https://dashboard.ngrok.com/get-started/setup).
     - `ngrok config add-authtoken <YOUR_NGROK_AUTH_TOKEN>`
@@ -124,42 +121,50 @@ NOTE: These steps will cover more aspects over time. For now it focuses on testi
     - Reload your server for the new env vars to apply and resend the verification payloads
     - You will have to do this with multiple settings, thus, you will have to repeat reloading your server as you add new variables
 
-1. Create a new personal GitHub organization at `<YOUR_GITHUB_USERNAME>/eng-pipes-org`
+1. [Create a GitHub organization](https://github.com/account/organizations/new?plan=free). Name it what you like.
 
-    - In that organization, create a [new GitHub App](https://github.com/settings/apps/new)
-    - Set the webhook to your ngrok tunnel with the GH route (e.g. `<NGROK_INSTANCE>/webhooks/github`)
-    - Place the secrets in your `.env` (see [Setup Secrets](#setup-secrets) below)
-    - Go to the `Permissions & events` sidebar menu entry of the GitHub app configuration, and grant maximum non-`Admin` access (`Read and write` where possible, `Read only` everywhere else) for every line in `Repository permissions` (NOTE: We use a more constrained permission-set in production, but for initial setup enabling maximal permissions is fine; permissions can be whittled down later as needed.)
-    - For `Organization permissions`, grant `Read and write` for `Members` and `Projects`
-    - In the `Subscribe to events` section, check every possible box
-    - Go to `Install App` in the sidebar menu of the GitHub app configuration, and install the app for your personal GitHub organization
-    - When prompted, choose `All repositories`
+    - In your organization, create a new GitHub repository named something like `testing-eng-pipes`.
+    - After the organization has been created, go to its `Settings`, then select `Personal access tokens` from the sidebar to enable tokens.
+    - In the setup menu, select `Allow access via fine-grained personal access tokens`, then `Do not require administrator approval`, and finally `Allow access via personal access tokens (classic)`.
 
-1. In your personal organization, create a new personal GitHub repository at `eng-pipes-org/eng-pipes-dev`.
+1.  [Create a personal access token](https://github.com/settings/tokens/new).
 
-    - After the organization has been created, go to its `Settings`, then select `Personal access tokens` from the sidebar to enable tokens
-    - In the setup menu, select `Allow access via fine-grained personal access tokens`, then `Do not require administrator approval`, and finally `Allow access via personal access tokens (classic)`
-    - Now, go to your own account `Settings`, select `Developer Settings` from the sidebar, then choose to `Generate personal access token (classic)`
     - Title the new token `Eng-pipes development token`, give this token 90 days until expiration, and enable the following permissions: `read:org` and `read:user`.
     - On the next page, copy the displayed token into the `GH_USER_TOKEN` field of your `.env` file.
 
-    > :warning: **You are giving this token rather broad permissions to your entire GitHub account, not just the `eng-pipes-org`, so be very careful and ensure it does not leave your machine!**
+      > :warning: **You are giving this token permissions to all orgs across GitHub that you are a member of (though some, like `getsentry`, are configured to require approval before PATs have access)). Be careful and ensure it does not leave your machine!**
+
+1.  [Create a GitHub App](https://github.com/settings/apps/new).
+
+    - Set the webhook to your ngrok tunnel with the GH route (e.g. `<NGROK_INSTANCE>/webhooks/github`)
+    - Create and download a private key and add it to your `.env` under `GH_APP_PRIVATE_KEY`. You'll need to strip newlines (or convert them to literal `\n`). (See [Setup Secrets](#setup-secrets) above.)
+    - Go to the `Permissions & events` sidebar menu entry of the GitHub app configuration, and grant maximum non-`Admin` access (`Read and write` where possible, `Read only` everywhere else) for every line in `Repository permissions` (NOTE: We use a more constrained permission-set in production, but for initial setup enabling maximal permissions is fine; permissions can be whittled down later as needed.)
+    - For `Organization permissions`, grant `Read and write` for `Members` and `Projects`
+    - In the `Subscribe to events` section, check every possible box
+    - Go to `Install App` in the sidebar menu of the GitHub app configuration, and install the app for your GitHub organization.
+    - When prompted, choose `All repositories`.
 
-1. In your personal GitHub organization, create a new project called `Eng-pipes test project`
+1. In your GitHub organization, create a new project called `GitHub Issues Someone Else Cares About`.
 
-    - Go to the project's `Settings`, then modify the `Status` field to only have the following options (note the capitalization): `Waiting for: Community`, `Waiting for: Product Owner`, and `Waiting for: Support`
-    - Add a new field (note the capitalization) called `Respond By` of type `Text`
-    - Add a new field (note the capitalization) called `Product Area` of type `Single select`, with the following options: `Alerts`, `Crons`, `Dashboards`, `Discover`, `Issues,` `Performance`, `Profiling`, `Projects`, `Relays`, `Releases`, and `User Feedback`
+    - Go to the project's `Settings`, then modify the `Status` field to only have the following options (note the capitalization): `Waiting for: Community`, `Waiting for: Product Owner`, and `Waiting for: Support`.
+    - Add a new field (note the capitalization) called `Response Due` of type `Text`.
+    - Add a new field (note the capitalization) called `Product Area` of type `Single select`, with the following options: `Alerts`, `Crons`, `Dashboards`, `Discover`, `Issues,` `Performance`, `Profiling`, `Projects`, `Relays`, `Releases`, and `User Feedback`.
 
-1. In your personal GitHub repository at `eng-pipes-org/eng-pipes-dev`, go to `Issues`, then click `Labels`
+1. In your GitHub repository at `[your-org]/testing-eng-pipes-or-whatever`, go to `Issues`, then click `Labels`.
 
     - Add the labels `Waiting for: Community`, `Waiting for: Product Owner`, and `Waiting for: Support`,
     - Add the labels `Product Area: Alerts`, `Product Area: Crons`, `Product Area: Dashboards`, `Product Area: Discover`, `Product Area: Issues,` `Product Area: Performance`, `Product Area: Profiling`, `Product Area: Projects`, `Product Area: Relays`, `Product Area: Releases`, and `Product Area: User Feedback`
 
-1. In a terminal, log into the Github CLI using `gh auth login` and following the prompts
+1. Copy the file `github-orgs.example.yml` to `github-orgs.local.yml`.
+
+    - Modify it with the slug of your organization and the ID of your app.
+
+    - Leave the `privateKey` as-is, it's the name of an environment variable to pull from (the main `github-orgs.yml` holds public config and is checked into version control).
+
+    - In a terminal, log into the Github CLI using `gh auth login`.
 
-    - Use [this](https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/using-the-api-to-manage-projects#finding-the-node-id-of-an-organization-project) GraphQL query to identify the node ID of your personal GitHub organization; set the `ISSUES_PROJECT_NODE_ID` variable in your `.env` file to match
-    - Use [this](https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/using-the-api-to-manage-projects#finding-the-node-id-of-a-field) GraphQL query to identify the node ID of your project fields, and use those IDs to populate the `STATUS_FIELD_ID`, `PRODUCT_AREA_FIELD_ID`, and `RESPONSE_DUE_DATE_FIELD_ID` in your `.env` file
+    - Use [this](https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/using-the-api-to-manage-projects#finding-the-node-id-of-an-organization-project) GraphQL query to identify the node ID of the project you made earlier; set `nodeId` to match.
+    - Use [this](https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/using-the-api-to-manage-projects#finding-the-node-id-of-a-field) GraphQL query to identify the IDs of the project fields you set up, and use those to populate `fieldIds`.
 
 1. Follow the steps of the "Development & tests" section below to get the server running.
 
@@ -174,15 +179,15 @@ NOTE: These steps will cover more aspects over time. For now it focuses on testi
 
 1. Verify that the GitHub -> eng-pipes server pipeline works
 
-    - In your `eng-pipes-dev` GitHub repository, create a new issue
+    - In your `testing-eng-pipes` GitHub repository, create a new issue
     - You should see your localhost app response with a 200 status code:
 
     ![success!](/docs/successful_github_event.png 'Successful GitHub webhook reception')
 
 1. Verify that the GitHub -> eng-pipes server -> Slack pipeline works
 
     - In your Slack workspace, create a `#test` channel, and in that channel type `/notify-for-triage Alerts sfo` to setup Slack messages
-    - Create a new issue in `eng-pipes-org/eng-pipes-dev`, then add the `Product Area: Alerts` label to it
+    - Create a new issue in `[your-org]/testing-eng-pipes`, then add the `Product Area: Alerts` label to it
     - You should see a 200 status code in the ngrok window, plus a description of the message in the `yarn dev` window, and a Slack message describing the new issue in the `#test` channel of your Slack workspace
 
 Congratulations, you're all setup!

github-orgs.example.yml

@@ -0,0 +1,10 @@
+your-org-CHANGEME:
+  appAuth:
+    appId: ''
+    privateKey: 'GH_APP_PRIVATE_KEY' # this is the name of an env var to pull from
+  project:
+    nodeId: ''
+    fieldIds:
+      productArea: ''
+      status: ''
+      responseDue: ''

github-orgs.yml

@@ -0,0 +1,10 @@
+getsentry:
+  appAuth:
+    appId: 66573
+    privateKey: 'GH_APP_PRIVATE_KEY_FOR_GETSENTRY'
+  project:
+    nodeId: 'PVT_kwDOABVQ184AOGW8'
+    fieldIds:
+      productArea: 'PVTSSF_lADOABVQ184AOGW8zgJEBno'
+      status: 'PVTSSF_lADOABVQ184AOGW8zgI_7g0'
+      responseDue: 'PVTF_lADOABVQ184AOGW8zgLLxGg'

src/api/github/__mocks__/octokitWithRetries.ts

@@ -97,7 +97,7 @@ OctokitWithRetries.issues = {
 OctokitWithRetries.orgs = {
   checkMembershipForUser: jest.fn(async function ({ org, username }) {
     let status = 302;
-    if (org === 'Enterprise' || org === null) {
+    if (org === 'getsentry' || org === null) {
       if (username === 'Picard' || username === 'Troi') {
         return { status: 204 };
       } else {
@@ -156,7 +156,7 @@ OctokitWithRetries.repos = {
 OctokitWithRetries.request = jest.fn(async (URL, x) => {
   if (URL.includes && URL.includes('/GTM/')) {
     let status = 302;
-    if (x.org === 'Enterprise' || x.org === null) {
+    if (x.org === 'getsentry' || x.org === null) {
       if (x.username === 'Troi') {
         return { status: 200 };
       } else {

src/api/github/org.test.ts

@@ -8,7 +8,7 @@ import { GitHubOrg } from './org';
 describe('constructor', function () {
   beforeAll(async function () {
     octokitClass.mockClear();
-    await new GitHubOrg({
+    new GitHubOrg('zerb', {
       appAuth: {
         appId: 423,
         privateKey: 'so secret',
@@ -37,8 +37,7 @@ describe('constructor', function () {
 
 describe('bindAPI', function () {
   beforeAll(async function () {
-    const org = await new GitHubOrg({
-      slug: 'banana',
+    const org = new GitHubOrg('banana', {
       appAuth: {
         appId: 422,
         privateKey: 'so private',

src/api/github/org.ts

@@ -27,8 +27,8 @@ export class GitHubOrg {
   // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation#using-the-octokitjs-sdk-to-authenticate-as-an-app-installation
   api: Octokit;
 
-  constructor(config: GitHubOrgConfig) {
-    this.slug = config.slug;
+  constructor(orgSlug: string, config: GitHubOrgConfig) {
+    this.slug = orgSlug;
     this.appAuth = config.appAuth;
     this.project = config.project;
 

src/brain/githubMetrics/index.test.ts

@@ -34,7 +34,7 @@ jest.spyOn(dbFunctions, 'insert');
 jest.spyOn(dbFunctions, 'insertOss');
 jest.mock('@/config', () => {
   const actualEnvVariables = jest.requireActual('@/config');
-  return { ...actualEnvVariables, DRY_RUN: true };
+  return { ...actualEnvVariables, DRY_RUN: false };
 });
 
 const SCHEMA = Object.entries(dbFunctions.TARGETS.oss.schema).map(
@@ -131,7 +131,7 @@ describe('github webhook', function () {
         action: 'opened',
         created_at: '2019-05-15T15:20:18Z',
         object_id: 1,
-        repository: 'Enterprise/Hello-World',
+        repository: 'getsentry/Hello-World',
         type: 'issues',
         updated_at: '2019-05-15T15:20:18Z',
         user_id: 21031067,
@@ -150,7 +150,7 @@ describe('github webhook', function () {
       label: {
         id: 1362934389,
         node_id: 'MDU6TGFiZWwxMzYyOTM0Mzg5',
-        url: 'https://api.github.com/repos/Enterprise/Hello-World/labels/bug',
+        url: 'https://api.github.com/repos/getsentry/Hello-World/labels/bug',
         name: 'bug',
         color: 'd73a4a',
         default: true,
@@ -171,7 +171,7 @@ describe('github webhook', function () {
         action: 'labeled',
         created_at: '2019-05-15T15:20:18Z',
         object_id: 1,
-        repository: 'Enterprise/Hello-World',
+        repository: 'getsentry/Hello-World',
         type: 'issues',
         updated_at: '2019-05-15T15:20:18Z',
         user_id: 21031067,
@@ -205,7 +205,7 @@ describe('github webhook', function () {
         action: 'opened',
         created_at: '2019-05-15T15:20:18Z',
         object_id: 1,
-        repository: 'Enterprise/Hello-World',
+        repository: 'getsentry/Hello-World',
         type: 'issues',
         updated_at: '2019-05-15T15:20:18Z',
         user_id: 21031067,

src/brain/issueLabelHandler/index.test.ts

@@ -114,7 +114,7 @@ describe('issueLabelHandler', function () {
       sender: { login: sender || 'Skywalker' }, // default to external user
       repository: {
         name: repo,
-        owner: { login: 'Enterprise' },
+        owner: { login: GETSENTRY_ORG.slug },
       },
       issue: { state, labels }, // mix in labels stored in mock
     };

src/brain/issueNotifier/index.test.ts

@@ -96,7 +96,7 @@ describe('issueNotifier Tests', function () {
         expect(bolt.client.chat.postMessage).toBeCalledTimes(NUM_CHANNELS);
         for (let i = 1; i <= NUM_CHANNELS; i++) {
           expect(bolt.client.chat.postMessage).toHaveBeenCalledWith({
-            text: '⏲ A wild issue has appeared! <https://github.com/Enterprise/Hello-World/issues/1|#1 Spelling error in the README file>',
+            text: '⏲ A wild issue has appeared! <https://github.com/getsentry/Hello-World/issues/1|#1 Spelling error in the README file>',
             channel: channelId(i),
             unfurl_links: false,
             unfurl_media: false,
@@ -126,7 +126,7 @@ describe('issueNotifier Tests', function () {
       });
       expect(bolt.client.chat.postMessage).toHaveBeenLastCalledWith({
         channel: 'CHNLIDRND2',
-        text: '⏲ A wild issue has appeared! <https://github.com/Enterprise/Hello-World/issues/1|#1 &lt;Title with &lt; and &gt; characters&gt;>',
+        text: '⏲ A wild issue has appeared! <https://github.com/getsentry/Hello-World/issues/1|#1 &lt;Title with &lt; and &gt; characters&gt;>',
         unfurl_links: false,
         unfurl_media: false,
       });
@@ -146,7 +146,7 @@ describe('issueNotifier Tests', function () {
       expect(bolt.client.chat.postMessage).toBeCalledTimes(0);
       expect(bolt.client.chat.postMessage).not.toHaveBeenLastCalledWith({
         channel: 'C02KHRNRZ1B',
-        text: '⏲ Issue ready to route: <https://github.com/Enterprise/Hello-World/issues/1|#1 Spelling error in the README file>',
+        text: '⏲ Issue ready to route: <https://github.com/getsentry/Hello-World/issues/1|#1 Spelling error in the README file>',
         unfurl_links: false,
         unfurl_media: false,
       });
@@ -168,7 +168,7 @@ describe('issueNotifier Tests', function () {
       expect(bolt.client.chat.postMessage).toBeCalledTimes(1);
       expect(bolt.client.chat.postMessage).toHaveBeenLastCalledWith({
         channel: 'C02KHRNRZ1B',
-        text: '⏲ Issue ready to route: <https://github.com/Enterprise/Hello-World/issues/1|#1 Spelling error in the README file>',
+        text: '⏲ Issue ready to route: <https://github.com/getsentry/Hello-World/issues/1|#1 Spelling error in the README file>',
         unfurl_links: false,
         unfurl_media: false,
       });

src/brain/projectsHandler/index.test.ts

@@ -64,7 +64,7 @@ describe('projectsHandler', function () {
     fieldNodeId?: string
   ) {
     const projectPayload = {
-      organization: { login: 'test-org' },
+      organization: { login: GETSENTRY_ORG.slug },
       projects_v2_item: {
         project_node_id: projectNodeId || 'test-project-node-id',
         node_id: 'test-node-id',