feat: embedded db (#309)

* feat: Add support for embedded db

* feat: support embedded database in Helm chart

* fix: improve docker file

* fix: templates/deployment.yaml

* feat: ability to create postgresql stateful set

* fix: calling db.MustInit twice on operator mode

.dockerignore

@@ -0,0 +1,15 @@
+.bin/
+.config-db/
+.github/
+.vscode/
+
+build
+
+chart/
+CONTRIBUTING.md
+SECURITY.md
+README.md
+PROJECT
+
+cover.out
+.releaserc

Makefile

@@ -20,7 +20,7 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 docker:
-	docker build . -t ${IMG}
+	docker build . -f build/Dockerfile -t ${IMG}
 
 # Push the docker image
 docker-push:

Dockerfile → build/Dockerfile

@@ -1,13 +1,13 @@
 FROM golang:1.20@sha256:bc5f0b5e43282627279fe5262ae275fecb3d2eae3b33977a7fd200c7a760d6f1 as builder
 WORKDIR /app
-COPY ./ ./
 
 ARG VERSION
+
 COPY go.mod /app/go.mod
 COPY go.sum /app/go.sum
 RUN go mod download
-WORKDIR /app
-RUN go version
+
+COPY ./ ./
 RUN make build
 
 FROM ubuntu:jammy@sha256:0bced47fffa3361afa981854fcabcd4577cd43cebbb808cea2b1f33a3dd7f508
@@ -16,5 +16,11 @@ WORKDIR /app
 COPY --from=builder /app/.bin/config-db /app
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
+RUN mkdir /opt/database && groupadd --gid 1000 catalog && \
+  useradd catalog --uid 1000 -g catalog -m -d /var/lib/catalog && \
+  chown -R 1000:1000 /opt/database && chown -R 1000:1000 /app
+
+USER catalog:catalog
+
 RUN /app/config-db go-offline
-ENTRYPOINT ["/app/config-db"]
+ENTRYPOINT ["/app/config-db"]

chart/Chart.yaml

@@ -4,6 +4,6 @@ description: A Helm chart for config-db
 
 type: application
 
-version: 0.2.0
+version: 0.3.0
 
 appVersion: "0.0.5"

chart/templates/deployment.yaml

@@ -1,17 +1,41 @@
+{{$embeddedDB := and (eq .Values.db.external.enabled false) (eq .Values.db.embedded.persist true)  }}
+---
 apiVersion: apps/v1
+{{- if $embeddedDB}}
+kind: StatefulSet
+{{- else }}
 kind: Deployment
+{{- end }}
 metadata:
   name: {{ include "config-db.name" . }}
   labels: {{- include "config-db.labels" . | nindent 4 }}
 spec:
   replicas: {{ .Values.replicas }}
   selector:
     matchLabels: {{- include "config-db.selectorLabels" . | nindent 6 }}
+  {{- if $embeddedDB }}
+  serviceName: {{ include "config-db.name" . }}
+  volumeClaimTemplates:
+    - metadata:
+        name: config-db-embedded-database
+        labels:
+          {{- include "config-db.labels" . | nindent 10 }}
+      spec:
+        {{- if not (eq .Values.db.embedded.storageClass "") }}
+        storageClassName: {{ .Values.db.embedded.storageClass }}
+        {{- end }}
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: {{ .Values.db.embedded.storage }}
+  {{- end }}
   template:
     metadata:
       labels: {{- include "config-db.selectorLabels" . | nindent 8 }}
     spec:
       serviceAccountName: {{ template "config-db.serviceAccountName" . }}
+      securityContext:
+        fsGroup: 1000
       containers:
         - name: {{ include "config-db.name" . }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -42,7 +66,7 @@ spec:
             - --disable-postgrest={{ .Values.disablePostgrest }}
             - --change-retention-days={{ .Values.configChangeRetentionDays }}
             - --analysis-retention-days={{ .Values.configAnalysisRetentionDays }}
-            {{- if .Values.db.enabled}}
+            {{- if .Values.db.runMigrations}}
             - --db-migrations
             {{- end}}
           {{- if .Values.upstream.enabled}}
@@ -52,10 +76,14 @@ spec:
           {{- end}}
           env:
             - name: DB_URL
+              {{- if .Values.db.external.enabled}}
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Values.db.secretKeyRef.name }}
-                  key: {{ .Values.db.secretKeyRef.key }}
+                  name: {{ .Values.db.external.secretKeyRef.name }}
+                  key: {{ .Values.db.external.secretKeyRef.key }}
+              {{- else}}
+              value: "embedded:///opt/database"
+              {{- end}}
             - name: NAMESPACE
               value: {{ .Values.namespace | default .Release.Namespace }}
             {{- if .Values.upstream.enabled}}
@@ -66,6 +94,11 @@ spec:
             {{- end}}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            {{- if $embeddedDB}}
+            - name: config-db-embedded-database
+              mountPath: "/opt/database"
+            {{- end }}
       {{- with .Values.extra }}
       {{- toYaml . | nindent 6 }}
       {{- end }}

chart/templates/postgres.yaml

@@ -1,5 +1,4 @@
-{{- if eq .Values.db.enabled true }}
-
+{{- if .Values.db.external.create }}
 ---
 # PostgreSQL StatefulSet
 apiVersion: apps/v1
@@ -25,18 +24,18 @@ spec:
             mountPath: /data
         envFrom:
           - secretRef:
-              name: {{ .Values.db.secretKeyRef.name }}
+              name: {{ .Values.db.external.secretKeyRef.name }}
   volumeClaimTemplates:
     - metadata:
         name: postgresql
       spec:
         accessModes: ["ReadWriteOnce"]
-        {{- if not (eq .Values.db.storageClass "") }}
-        storageClassName: {{ .Values.db.storageClass }}
+        {{- if ne .Values.db.external.storageClass "" }}
+        storageClassName: {{ .Values.db.external.storageClass }}
         {{- end }}
         resources:
           requests:
-            storage: {{ .Values.db.storage }}
+            storage: {{ .Values.db.external.storage }}
 ---
 # PostgreSQL StatefulSet Service
 apiVersion: v1
@@ -50,11 +49,10 @@ spec:
   - port: 5432
     targetPort: 5432
 ---
-{{- if .Values.db.secretKeyRef.create }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ .Values.db.secretKeyRef.name }}
+  name: {{ .Values.db.external.secretKeyRef.name }}
   annotations:
     "helm.sh/resource-policy": "keep"
 type: Opaque
@@ -66,16 +64,12 @@ stringData:
   {{- $dbname := (( get $secretData "POSTGRES_DB" ) | b64dec ) | default "config_db" }}
   {{- $host := print (include "config-db.name" .) "-postgresql." .Release.Namespace  ".svc.cluster.local:5432" }}
   {{- $url := print "postgresql://" $user ":" $password "@" $host }}
-  {{- $configDbUrl := ( get $secretData .Values.db.secretKeyRef.key ) | default ( print $url "/config_db?sslmode=disable" ) }}
-
+  {{- $configDbUrl := ( get $secretData .Values.db.external.secretKeyRef.key ) | default ( print $url "/config_db?sslmode=disable" ) }}
   POSTGRES_USER: {{ $user | quote }}
   POSTGRES_PASSWORD: {{ $password | quote }}
   POSTGRES_HOST: {{ $host | quote }}
   POSTGRES_DB: {{ $dbname | quote }}
-  {{ .Values.db.secretKeyRef.key }}: {{ $configDbUrl | quote }}
-
-{{- end }}
-
+  {{ .Values.db.external.secretKeyRef.key }}: {{ $configDbUrl | quote }}
 ---
 
 {{- end }}

chart/values.yaml

@@ -4,34 +4,55 @@
 replicas: 1
 
 # Use this only if you want to replace the default that is .Chart.Name as the name of all the objects.
-nameOverride: ""
+nameOverride: ''
 
 # Set to true if you want to disable the postgrest service
-disablePostgrest: false 
+disablePostgrest: false
 
 image:
   repository: docker.io/flanksource/config-db
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "latest"
+  tag: 'latest'
 
 configChangeRetentionDays: 60
 configAnalysisRetentionDays: 60
 
 # a list of configmaps to load scrape rules from, the configmap should have a single entry called "config.yaml"
 scrapeRuleConfigMaps:
   - config-db-rules
+
 db:
-  # Setting this to true will create a postgres stateful set for config-db to connect to.
-  enabled: true
-  secretKeyRef:
-    create: true
-    # The name of the secret to look for.
-    name: config-db-postgresql
-    # This is the key that we look for in the secret.
-    key: DB_URL
-  storageClass: ""
-  storage: 20Gi
+  runMigrations: true
+  embedded:
+    # If the database is embedded, setting this to true will persist the contents of the database
+    # through a persistent volume
+    persist: true
+    storageClass: ''
+    storage: 20Gi
+  external:
+    # Setting enabled to true will use an external postgres DB.
+    # You can either use the embedded db or an external db.
+    # If both is enabled, then embedded db will take precedence.
+    enabled: false
+    # Setting create:true will create
+    #   - a postgres stateful set
+    #   - the secret &
+    #   - the service to expose the postgres stateful set.
+    # By default, the generated secret will use 'postgres' as the username and a randomly generated password.
+    # If you need to set a custom username and password, you can populate a secret named 'postgres-connection' before install
+    # with POSTGRES_USER and POSTGRES_PASSWORD
+    #
+    # If create:false, a prexisting secret containing the URI to an existing postgres database must be provided
+    # The URI must be in the format 'postgresql://"$user":"$password"@"$host"/"$database"'
+    create: false
+    secretKeyRef:
+      # The name of the secret to look for.
+      name: config-db-postgresql
+      # This is the key that we look for in the secret.
+      key: DB_URL
+    storageClass: ''
+    storage: 20Gi
 
 ingress:
   enabled: false
@@ -55,7 +76,7 @@ resources:
 
 serviceAccount:
   create: true
-  name: ""
+  name: ''
   annotations: {}
 
 upstream:

cmd/operator.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"os"
 
@@ -39,8 +38,10 @@ func init() {
 }
 
 func run(cmd *cobra.Command, args []string) {
-	db.MustInit()
-	api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool)
+	ctx := cmd.Context()
+
+	db.MustInit(ctx)
+	api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool)
 
 	zapLogger := logger.GetZapLogger()
 	if zapLogger == nil {
@@ -64,7 +65,7 @@ func run(cmd *cobra.Command, args []string) {
 	utilruntime.Must(configsv1.AddToScheme(scheme))
 
 	// Start the server
-	go serve(args)
+	go serve(ctx, args)
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,

cmd/run.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -23,15 +22,17 @@ var Run = &cobra.Command{
 	Use:   "run <scraper.yaml>",
 	Short: "Run scrapers and return",
 	Run: func(cmd *cobra.Command, configFiles []string) {
+		ctx := cmd.Context()
+
 		logger.Infof("Scraping %v", configFiles)
 		scraperConfigs, err := v1.ParseConfigs(configFiles...)
 		if err != nil {
 			logger.Fatalf(err.Error())
 		}
 
 		if db.ConnectionString != "" {
-			db.MustInit()
-			api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool)
+			db.MustInit(ctx)
+			api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool)
 		}
 
 		if db.ConnectionString == "" && outputDir == "" {

cmd/server.go

@@ -23,13 +23,15 @@ import (
 var Serve = &cobra.Command{
 	Use: "serve",
 	Run: func(cmd *cobra.Command, args []string) {
-		serve(args)
+		ctx := cmd.Context()
+
+		db.MustInit(ctx)
+		serve(ctx, args)
 	},
 }
 
-func serve(configFiles []string) {
-	db.MustInit()
-	api.DefaultContext = api.NewScrapeContext(context.Background(), db.DefaultDB(), db.Pool)
+func serve(ctx context.Context, configFiles []string) {
+	api.DefaultContext = api.NewScrapeContext(ctx, db.DefaultDB(), db.Pool)
 
 	e := echo.New()
 	// PostgREST needs to know how it is exposed to create the correct links
@@ -60,8 +62,19 @@ func serve(configFiles []string) {
 
 	go jobs.ScheduleJobs()
 
-	if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil {
-		e.Logger.Fatal(err)
+	go func() {
+		if err := e.Start(fmt.Sprintf(":%d", httpPort)); err != nil {
+			e.Logger.Fatal(err)
+		}
+	}()
+
+	<-ctx.Done()
+	if err := db.StopEmbeddedPGServer(); err != nil {
+		logger.Errorf("failed to stop server: %v", err)
+	}
+
+	if err := e.Shutdown(ctx); err != nil {
+		logger.Errorf("failed to shutdown echo HTTP server: %v", err)
 	}
 }