Quip Sync For Docs (#76)

* v1

* basic sync

* comments

* x

* ruby 3.2.5

* version

* v

* v

* quip status

* unique

* status

* sync indicator

.github/workflows/rubyonrails.yml

@@ -30,7 +30,7 @@ jobs:
         uses: actions/checkout@v4
       # Add or replace dependency steps here
       - name: Install Ruby and gems
-        uses: ruby/setup-ruby@55283cc23133118229fd3f97f9336ee23a179fcf # v1.146.0
+        uses: ruby/setup-ruby@v1 # v1.146.0
         with:
           bundler-cache: true
 
@@ -47,8 +47,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install Ruby and gems
-        uses: ruby/setup-ruby@55283cc23133118229fd3f97f9336ee23a179fcf # v1.146.0
+        uses: ruby/setup-ruby@v1
         with:
+          ruby-version: '3.2.2'
           bundler-cache: true
       # Add or replace any other lints here
       - name: Security audit dependencies

Gemfile

@@ -119,4 +119,4 @@ gem 'bundle-audit'
 
 gem 'brakeman'
 
-gem 'reverse_markdown'
+gem 'reverse_markdown'

Gemfile.lock

@@ -361,6 +361,7 @@ GEM
 
 PLATFORMS
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-22
   x86_64-linux
 

app/controllers/base_documents_controller.rb

@@ -82,13 +82,6 @@ def create
 
     respond_to do |format|
       if @document.save
-        # This will space out embeddings when jobs start to back up, especially during an import
-        # TODO: make this more configurable
-        total_jobs = Delayed::Job.count
-        delay_seconds = total_jobs * 3 # 3 second delay per job in the queue
-
-        EmbedDocumentJob.set(priority: 5, wait: delay_seconds.seconds).perform_later(@document.id) if @document.previous_changes.include?('check_hash')
-
         format.html do
           redirect_to document_url(@document), notice: 'Document was successfully created.'
         end
@@ -120,6 +113,6 @@ def set_document
 
   # Only allow a list of trusted parameters through.
   def document_params
-    params.require(:document).permit(:document, :title, :enabled, :external_id, :url, :library_id)
+    params.require(:document).permit(:document, :title, :enabled, :external_id, :url, :library_id, :source_url)
   end
 end

app/jobs/sync_quip_doc_job.rb

@@ -0,0 +1,48 @@
+class SyncQuipDocJob < ApplicationJob
+  queue_as :default
+
+  def perform(_doc_id)
+    begin
+      document = Document.find(_doc_id)
+    rescue ActiveRecord::RecordNotFound => e
+      # Handle the case where the document is not found
+      Rails.logger.error("Document with id #{_doc_id} not found: #{e.message}")
+      return # Exit early since there's nothing to sync
+    end
+
+    # Log or handle cases where the document has no quip_url
+    if document.source_url.blank?
+      Rails.logger.warn("Document with id #{_doc_id} has no source URL.")
+      return
+    end
+
+    begin
+      # Initialize the Quip client
+      quip_client = Quip::Client.new(access_token: ENV.fetch('QUIP_TOKEN'))
+      uri = URI.parse(document.source_url)
+      path = uri.path.sub(%r{^/}, '') # Removes the leading /
+      quip_thread = quip_client.get_thread(path)
+
+      # Convert Quip HTML content to Markdown
+      markdown_quip = ReverseMarkdown.convert(quip_thread['html'])
+
+      document.document = markdown_quip
+      document.synced_at = DateTime.current
+      document.last_sync_result = 'SUCCESS'
+      document.save
+    rescue Quip::Error => e
+      # Handle Quip-specific errors
+      Rails.logger.error("Quip API error while fetching document from #{document.source_url}: #{e.message}")
+      document.last_sync_result = "#{e.message}"
+      document.save
+    rescue StandardError => e
+      # Handle any other unforeseen errors
+      Rails.logger.error("Unexpected error during sync for document id #{_doc_id}: #{e.message}")
+      document.last_sync_result = "#{e.message}"
+      document.save
+    end
+
+    # Reschedule the job to run again in 24 hours
+    SyncQuipDocJob.set(wait: 24.hours).perform_later(_doc_id)
+  end
+end

app/models/document.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 class Document < ApplicationRecord
   include PgSearch::Model
 
@@ -24,14 +22,29 @@ class Document < ApplicationRecord
   validate :token_count_must_be_less_than
 
   validates :external_id, uniqueness: true, if: -> { external_id.present? }
+  validates :source_url, uniqueness: true, if: -> { source_url.present? }
 
   validates :length, presence: true
 
   validates :document, presence: true,
-                       uniqueness: { scope: :check_hash, message: 'Document with same content already exists.' }
+                       uniqueness: { scope: :check_hash, message: 'Document with same content already exists.' },
+                       unless: -> { source_url.present? }
 
   before_validation :calculate_length, :calculate_tokens, :calculate_hash
 
+  after_save :sync_quip_doc_if_needed
+  after_commit :schedule_embed_document_job, if: -> { previous_changes.include?('check_hash') }
+
+  def source_type
+    if source_url.include?('quip.com')
+      'quip'
+    elsif source_url.present?
+      'other'
+    elsif source_url.blank?
+      'none'
+    end
+  end
+
   def calculate_length
     # Calculate the length of the 'document' column and store it in the 'length' column
     return unless document
@@ -70,4 +83,22 @@ def token_count_must_be_less_than
 
     errors.add(:token_count, "is #{token_count} and must be less than 8,000")
   end
+
+  # Sync the document with Quip if source_url is present, contains 'quip.com',
+  def sync_quip_doc_if_needed
+    return unless source_url.present? && source_url.include?('quip.com')
+
+    return unless synced_at.nil? # only schedule if it is for the initial sync
+
+    SyncQuipDocJob.perform_later(id)
+  end
+
+  # Schedule the EmbedDocumentJob with a delay based on the number of jobs in the queue
+  def schedule_embed_document_job
+    total_jobs = Delayed::Job.count
+    delay_seconds = total_jobs * 3 # 3-second delay per job in the queue
+
+    # Set the priority and delay, and queue the job if the check_hash has changed
+    EmbedDocumentJob.set(priority: 5, wait: delay_seconds.seconds).perform_later(id)
+  end
 end

app/views/delayed_jobs/index.html.erb

@@ -24,7 +24,7 @@
           </div>
         </div>
         <div class="mb-2">
-          <strong>Run At:</strong> <%= time_ago_in_words(job.run_at) %> ago
+          <strong>Run At:</strong> <%= "#{distance_of_time_in_words(Time.now, job.run_at)} from now" %>
         </div>
         <div class="mb-2">
           <strong>Created At:</strong> <%= time_ago_in_words(job.created_at) %> ago

app/views/documents/_document.html.erb

@@ -1,15 +1,35 @@
+<% if document.library_id %>
+  <p class="mb-2 flex items-center mr-2 text-stone-600">
+    <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 mr-1">
+      <path stroke-linecap="round" stroke-linejoin="round" d="M2.25 12.75V12A2.25 2.25 0 014.5 9.75h15A2.25 2.25 0 0121.75 12v.75m-8.69-6.44l-2.12-2.12a1.5 1.5 0 00-1.061-.44H4.5A2.25 2.25 0 002.25 6v12a2.25 2.25 0 002.25 2.25h15A2.25 2.25 0 0021.75 18V9a2.25 2.25 0 00-2.25-2.25h-5.379a1.5 1.5 0 01-1.06-.44z" />
+    </svg>
+    <%= link_to document.library.name, document.library, class:"text-sky-500" %>
+  </p>
+<% end %>
 <div id="<%= dom_id document %>" class="border">
   <div class="text-xs text-stone-500">
     <div class="flex justify-between p-3 <%= @document.enabled ? "bg-sky-800" : "bg-stone-400" %> rounded-t-lg">
       <h1 class="text-3xl font-light <%= @document.enabled ? "text-sky-100" : "text-stone-200" %>"><%= document.title %></h1>
     </div>
     <div class="flex justify-between items-center p-3 bg-stone-100">
       <div class="w-full">
-        <% if document.url %>
-          <p class="text-xs">Source <%= link_to document.url, document.url, class:"text-sky-500",target:"_source" %></p>
+        <p class="text-xs text-stone-500 font-semibold mt-1">
+          Created <%= time_ago_in_words(document.created_at) %> ago 
+          | Updated <%= time_ago_in_words(document.updated_at) %> ago
+          <%= "| Synced #{time_ago_in_words(document.synced_at)} ago" if document.synced_at.present? %>
+        </p>
+        <% if document.source_url %>
+          <div class="flex items-center space-x-2 text-xs mt-2">
+            <span>Source</span>
+            <%= link_to document.source_url, document.source_url, class:"text-sky-500", target:"_source" %>
+            <% if document.source_type == 'quip' %>
+              <%= image_tag 'quip_logo.png', alt: 'Quip Logo', class: 'h-6 mb-1' %>
+              <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-5 mb-2 <%= "text-green-500" if document.last_sync_result == "SUCCESS" %>">
+                <path stroke-linecap="round" stroke-linejoin="round" d="M16.023 9.348h4.992v-.001M2.985 19.644v-4.992m0 0h4.992m-4.993 0 3.181 3.183a8.25 8.25 0 0 0 13.803-3.7M4.031 9.865a8.25 8.25 0 0 1 13.803-3.7l3.181 3.182m0-4.991v4.99" />
+              </svg>
+            <% end %>
+          </div>
         <% end %>
-        <p class="text-xs">Library <%= link_to document.library.name, document.library, class:"text-sky-500" if document.library_id %></p>
-        <p class="text-xs text-stone-800 mt-1">Created <%= time_ago_in_words(document.created_at) %> ago | Updated <%= time_ago_in_words(document.updated_at) %> ago</p>
       </div>
       <% if policy(@document).edit? %>
         <div class="text-stone-800">

app/views/documents/_form.html.erb

@@ -14,7 +14,11 @@
     <%= form.text_field :title, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 my-2 w-full" %>
   </div>
   <div class="my-5">
-    <%= form.label :document, style: "display: block" %>
+    <%= form.label :source_url, "Source URL. Quip URLs will be synced daily.", class: "block text-gray-700 font-medium" %>
+    <%= form.text_field :source_url, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 my-2 w-full" %>
+  </div>
+  <div class="my-5">
+    <%= form.label :document, "Body.  Can be blank if Quip Source URL provided.", style: "display: block" %>
     <%= form.text_area :document, rows: 10, class: "block shadow rounded-md border border-gray-200 outline-none px-3 py-2 my-2 w-full" %>
   </div>
   <div class="my-5">

db/migrate/20241015012935_add_source_url_to_document.rb

@@ -0,0 +1,5 @@
+class AddSourceUrlToDocument < ActiveRecord::Migration[7.1]
+  def change
+    add_column :documents, :source_url, :string
+  end
+end

db/migrate/20241015013228_add_synced_at_to_document.rb

@@ -0,0 +1,5 @@
+class AddSyncedAtToDocument < ActiveRecord::Migration[7.1]
+  def change
+    add_column :documents, :synced_at, :datetime
+  end
+end

db/migrate/20241015190422_add_last_sync_result_to_document.rb

@@ -0,0 +1,5 @@
+class AddLastSyncResultToDocument < ActiveRecord::Migration[7.1]
+  def change
+    add_column :documents, :last_sync_result, :string
+  end
+end