Skip to content

Commit

Permalink
Split workflow definitions into sensible packages
Browse files Browse the repository at this point in the history
  • Loading branch information
nfx committed Oct 30, 2024
1 parent 448ea6a commit c82cd0c
Show file tree
Hide file tree
Showing 71 changed files with 160 additions and 71 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class AutoScale(maxWorkers: Option[Int], minWorkers: Option[Int] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute
import com.databricks.sdk.service.compute.{AwsAvailability, EbsVolumeType}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute
import com.databricks.sdk.service.compute.AzureAvailability

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class ClientsTypes(jobs: Boolean = false, notebooks: Boolean) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.sources.{DbfsStorageInfo, S3StorageInfo}
import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class ClusterLogConf(dbfs: Option[DbfsStorageInfo], s3: Option[S3StorageInfo] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.labs.remorph.intermediate.workflows.libraries.Library
import com.databricks.sdk.service.jobs

case class ClusterSpec(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class DiskSpec(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute
import com.databricks.sdk.service.compute.{DiskTypeAzureDiskVolumeType, DiskTypeEbsVolumeType}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute
import com.databricks.sdk.service.compute.GcpAvailability

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.sources.{Adlsgen2Info, DbfsStorageInfo, GcsStorageInfo, LocalFileInfo, S3StorageInfo, VolumesStorageInfo, WorkspaceStorageInfo}
import com.databricks.labs.remorph.intermediate.workflows._
import com.databricks.sdk.service.compute

case class InitScriptInfo(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class LogAnalyticsInfo(logAnalyticsPrimaryKey: Option[String], logAnalyticsWorkspaceId: Option[String] = None)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.libraries.DockerImage
import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute
import com.databricks.sdk.service.compute.{DataSecurityMode, RuntimeEngine}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.clusters

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class WorkloadType(clients: ClientsTypes) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class Job(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.labs.remorph.intermediate.workflows.clusters.ClusterSpec
import com.databricks.sdk.service.jobs

case class JobCluster(jobClusterKey: String, newCluster: ClusterSpec) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.JobDeploymentKind

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobEmailNotifications(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.labs.remorph.intermediate.workflows.libraries.Environment
import com.databricks.sdk.service.jobs

case class JobEnvironment(environmentKey: String, spec: Option[Environment] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobNotificationSettings(noAlertForCanceledRuns: Boolean = false, noAlertForSkippedRuns: Boolean)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobParameter(default: Option[String], name: Option[String], value: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobParameterDefinition(name: String, default: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobRunAs(servicePrincipalName: Option[String], userName: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.schedules.{Continuous, CronSchedule, TriggerSettings}
import com.databricks.labs.remorph.intermediate.workflows.sources.GitSource
import com.databricks.labs.remorph.intermediate.workflows.tasks.Task
import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.labs.remorph.intermediate.workflows.webhooks.WebhookNotifications
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.{Format, JobEditMode}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.JobSourceDirtyState

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.{JobsHealthMetric, JobsHealthOperator}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.jobs

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class JobsHealthRules(rules: Seq[JobsHealthRule] = Seq.empty) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class DockerBasicAuth(password: Option[String], username: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class DockerImage(basicAuth: Option[DockerBasicAuth], url: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class Environment(client: String, dependencies: Seq[String] = Seq.empty) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class Library(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class MavenLibrary(coordinates: String, exclusions: Seq[String] = Seq.empty, repo: Option[String] = None)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class PythonPyPiLibrary(spec: String, repo: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.libraries

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class RCranLibrary(spec: String, repo: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.PauseStatus

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.PauseStatus

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class FileArrivalTriggerConfiguration(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.PeriodicTriggerConfigurationTimeUnit

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.Condition

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.schedules

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.PauseStatus

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class Adlsgen2Info(destination: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class DbfsStorageInfo(destination: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class GcsStorageInfo(destination: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.jobs

case class GitSnapshot(usedCommit: Option[String] = None) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.labs.remorph.intermediate.workflows.jobs.JobSource
import com.databricks.sdk.service.jobs
import com.databricks.sdk.service.jobs.GitProvider

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class LocalFileInfo(destination: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class S3StorageInfo(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class VolumesStorageInfo(destination: String) extends JobNode {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.databricks.labs.remorph.intermediate.workflows
package com.databricks.labs.remorph.intermediate.workflows.sources

import com.databricks.labs.remorph.intermediate.workflows.JobNode
import com.databricks.sdk.service.compute

case class WorkspaceStorageInfo(destination: String) extends JobNode {
Expand Down
Loading

0 comments on commit c82cd0c

Please sign in to comment.