Import landscape.cncf.io dataset, publish to Neo4j Aura #907
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Import landscape.cncf.io dataset, publish to Neo4j Aura | |
| on: | |
| schedule: | |
| - cron: 0 0 * * * | |
| workflow_dispatch: {} | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/flat.yml | |
| - clean-landscape-json.ts | |
| jobs: | |
| scheduled: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup deno | |
| uses: denoland/setup-deno@main | |
| with: | |
| deno-version: v1.10.x | |
| - name: Check out repo | |
| uses: actions/checkout@v2 | |
| - name: Fetch data | |
| uses: githubocto/flat@v3 | |
| with: | |
| http_url: https://landscape.cncf.io/data/items.json | |
| downloaded_filename: landscape-items.json | |
| postprocess: ./clean-landscape-json.ts | |
| - name: Import to Neo4j Aura | |
| uses: johnymontana/[email protected] | |
| with: | |
| neo4j-user: ${{secrets.NEO4J_AURA_USER}} | |
| neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}} | |
| neo4j-uri: ${{secrets.NEO4J_AURA_URI}} | |
| filename: 'landscape-items-clean.json' | |
| cypher-query: > | |
| UNWIND $value AS value | |
| MERGE (c:Card {name: value.name}) | |
| SET c += value { | |
| .bestPracticeBadgeId, | |
| .bestPracticePercentage, | |
| .commitsThisYear, | |
| .contributorsCount, | |
| .crunchbase, | |
| .description, | |
| .homepage_url, | |
| .isSubsidiaryProject, | |
| .logo, | |
| .oss, | |
| .repo_url, | |
| .stars, | |
| .twitter, | |
| .license, | |
| .member | |
| } | |
| WITH c, value | |
| MERGE(org:Organization { name: value.organization }) | |
| MERGE(c)-[:IN_ORGANIZATION]->(org) | |
| MERGE(p:Path {name: value.path}) | |
| MERGE(c)-[:IN_PATH]->(p) | |
| MERGE(cat:Category {name: value.category}) | |
| MERGE(c)-[:IN_CATEGORY]->(cat) | |
| MERGE(hq:Headquarters { name: value.headquarters }) | |
| MERGE(c)-[:BASED_IN]->(hq) | |
| WITH c, value | |
| UNWIND value.industries as industry | |
| MERGE (i:Industry {name: industry}) | |
| MERGE (c)-[:IN_INDUSTRY]->(i) | |
| WITH c, value | |
| UNWIND value.github_data.languages as language | |
| MERGE (l:Language {name: language.name}) | |
| MERGE (c)-[:USES_LANGUAGE]->(l) | |
| WITH c, value | |
| UNWIND value.repos as repo | |
| MERGE (r:Repo {url: repo.url}) // TODO: should parse out an org/repo id for this | |
| MERGE (c)-[:OWNS_REPO]->(r) | |
| RETURN c.name, c.homepage_url, c.description | |
| - name: Assign OSS Licenses to Projects | |
| uses: johnymontana/[email protected] | |
| with: | |
| neo4j-user: ${{secrets.NEO4J_AURA_USER}} | |
| neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}} | |
| neo4j-uri: ${{secrets.NEO4J_AURA_URI}} | |
| filename: 'landscape-items-clean.json' | |
| cypher-query: > | |
| MATCH(card:Card) | |
| WHERE card.license IS NOT NULL | |
| MERGE(proj:Project {name: card.name}) | |
| MERGE(ossl:License { name: card.license }) | |
| MERGE (c)-[:IS_PROJECT]->(proj) | |
| MERGE (p)-[:HAS_LICENSE]->(ossl) | |
| RETURN p.name, ossl.name | |
| - name: Assign Membership Types to Members | |
| uses: johnymontana/[email protected] | |
| with: | |
| neo4j-user: ${{secrets.NEO4J_AURA_USER}} | |
| neo4j-password: ${{secrets.NEO4J_AURA_PASSWORD}} | |
| neo4j-uri: ${{secrets.NEO4J_AURA_URI}} | |
| filename: 'landscape-items-clean.json' | |
| cypher-query: > | |
| MATCH(c:Card) | |
| WHERE c.member IS NOT NULL | |
| MERGE(m:Membership {name: c.member}) | |
| MERGE(c)-[:IS_MEMBERTYPE]->(m) | |
| RETURN c.name, m.name |