Merge pull request #62 from rjsears/0.95

0.95

README.md

@@ -1,7 +1,7 @@
  <h2 align="center">
   <a name="chia_drive_logo" href="https://github.com/rjsears/chia_plot_manager"><img src="https://github.com/rjsears/chia_plot_manager/blob/main/images/chia_plot_manager_new.png" alt="Chia Plot Manager"></a><br>
 
-  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.94 - August 8th, 2021)
+  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.95 - September 3rd, 2021)
   </h2>
   <p align="center">
 Multi Server Chia Plot and Drive Management Solution
@@ -98,7 +98,7 @@ Beginning in V0.94, we now fully support portable plot management as well as old
   <li>After the transfer is complete, checks the exact file size of the plot on both systems as a basic verification</li>
   <li>Once files sizes are verified, deletes the sent plot</li>
   <li>Kills any lingering netcat connections on the selected Harvester/NAS</li>
-  <li>Supports any number of harvesters and prioritizes sending plots to the harvester with the most space available.</li>
+  <li>Supports any number of harvesters and prioritizes sending plots to the harvester with the most space available (Empty plot space + old plots to replace = total space available.</li>
   <li>If replacing old plots with new pool plots, above utilizes number of plot spaces available based on free space + number of old plots.</li>
 </ul>
 <br>
@@ -152,7 +152,7 @@ I am running on Python 3.8.5 and pretty much everything else came installed with
   <li><a href="https://pypi.org/project/configparser/">ConfigParser (5.0.2)</a> - Used for reading and updating config files</li>
   <li><a href="https://github.com/truenas/py-SMART">py-SMART (0.3)</a> - Used for reading drive information</li>
   <li><a href="https://pypi.org/project/natsort/">Natsort (7.1.1)</a> - Used for natural sorting of drive numbers</li>
- <li><a href="https://github.com/sysstat/sysstat">Sysstat</a> - Used to monitor Disk I/O Stats</li>
+ <li><a href="https://github.com/sysstat/sysstat">Sysstat</a> - Used to monitor Disk and Network I/O Stats</li>
  <li><a href="http://www.paramiko.org/">Paramiko (2.7.2)</a> - Used to grab remote harvester stats</li>
 
  </ul>
@@ -838,6 +838,25 @@ strategy above, it is super easy to add more drives.
 <br><hr>
 
 ### <a name="changelog"></a>Changelog
+<b>V0.95 2021-09-03</b>
+   - Replaces glances with sysstat commands to check and verify network traffic
+   - Update install script to create new network check script based on install directory
+   - Created failsafe to allow free drive space to be filled when 'replace_plots' has been
+     set to yes and there are no more old plots to replace. Now falls back to filling empty
+     drive space after that happens and will notify you when both fail.
+   - Various bug fixes and enhancements.
+
+<b>V0.94 2021-08-08</b>
+   - Automatic plot removal and replacement of older style plots with new style pooling
+     plots done on a one-by-one basis to keep as many plots farming as possible during 
+     the plot replacement process.
+   - Various bug fixes and other small enhancements
+   - Updated Farming Wide reports to report on number of portable plots vs old plots
+     and if we are replacing old plots or not on a per-server basis.
+   - Updated nas_export to include the number of old plots and if server is set to replace
+     old plots, the total number of plot space available on server is now the total of the
+     number of old plots to replace along with how many plots we can store on remaining 
+     free space.
 
 <b>V0.93 2021-07-08</b>
    - Added ability to identify plots as `portable` (set `pooling: True` in config file)

chianas/config_file_updater.py

@@ -4,7 +4,7 @@
 """
 Part of drive_manager. Checks to see if our config file needs updating and updated it for us.
 """
-VERSION = "V0.94 (2021-08-08)"
+VERSION = "V0.95 (2021-09-03)"
 
 import os
 import yaml

chianas/drive_manager.py

@@ -3,7 +3,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = 'Richard J. Sears'
-VERSION = "0.94 (2021-08-08)"
+VERSION = "0.95 (2021-09-03)"
 
 """
 NOTE NOTE NOTE NOTE NOTE NOTE NOTE
@@ -186,6 +186,9 @@
 plot_size_g = 101.3623551
 receive_script = script_path.joinpath('receive_plot.sh')
 replace_plots_receive_script = script_path.joinpath('replace_plots_receive_plot.sh')
+remote_transfer_active_file = script_path.joinpath('remote_transfer_is_active')
+network_check = script_path.joinpath('check_network_io.sh')
+network_check_output = script_path.joinpath('network_stats.io')
 
 # Date and Time Stuff
 current_military_time = datetime.now().strftime('%Y%m%d%H%M%S')
@@ -663,6 +666,7 @@ def get_plot_drive_to_use():
         return (natsorted(available_drives)[0])
     except IndexError:
         log.debug("ERROR: No Drives Found, Please add drives, run auto_drive.py and try again!")
+        notify('OUT OF DRIVE SPACE!', 'You have run out of drive space on your Harvester and we can no longer accept any new plots!')
         exit()
 
 
@@ -696,9 +700,6 @@ def get_internal_plot_drive_to_use():
             log.debug('this issue is you are able.')
             notify('Drive Overlap!', 'Internal and External plotting drives now overlap! Suggest fixing to prevent drive bus contention and slow transfers. If you have selected plot replacement, we will attempt to convert to replacement now.')
             return get_plot_drive_to_use()
-       # log.debug("ERROR: No Additional Internal Drives Found, Please add drives, run auto_drive.py and try again!")
-       # exit()
-
 
 
 def get_sorted_drive_list():
@@ -866,14 +867,15 @@ def update_receive_plot():
     here. See TODO: Update to use netcat native to python.
     """
     log.debug("update_receive_plot() Started")
+    remote_transfer_active = check_for_active_remote_transfer()
     if not chianas.replace_non_pool_plots: # If we are not replacing old plots with new portable plots, run the following code
         log.debug('Replace Plots has NOT been set in config, will call build script for normal operation.')
         drive = get_plot_drive_to_use()[0]
         if not os.path.isfile(receive_script):
             log.debug(f'{receive_script} not found. Building it now...')
             build_receive_plot('normal', drive)
         else:
-            if os.path.isfile(script_path.joinpath('remote_transfer_is_active')):
+            if remote_transfer_active:
                 log.debug('Remote Transfer in Progress, will try again soon!')
                 quit() # TODO Think about what we really want to do here. If we are running a remote transfer, can we still do other things?
             else:
@@ -901,7 +903,7 @@ def update_receive_plot():
                     log.debug(f'{receive_script} not found. Building it now...')
                     build_receive_plot('normal', drive)
                 else:
-                    if os.path.isfile(script_path.joinpath('remote_transfer_is_active')):
+                    if remote_transfer_active:
                         log.debug('Remote Transfer in Progress, will try again soon!')
                         quit()  # TODO Think about what we really want to do here. If we are running a remote transfer, can we still do other things?
                     else:
@@ -912,8 +914,7 @@ def update_receive_plot():
                             log.debug(f'No changes necessary to {receive_script}')
                             log.debug(f'Plots left available on configured plotting drive: {get_drive_info("space_free_plots_by_mountpoint", chianas.current_plotting_drive)}')
                         else:
-                            send_new_plot_disk_email()  # This is the full Plot drive report. This is in addition to the generic email sent by the
-                            # notify() function.
+                            send_new_plot_disk_email()  # This is the full Plot drive report. This is in addition to the generic email sent by the notify() function.
                             notify('Plot Drive Updated', f'Plot Drive Updated: Was: {chianas.current_plotting_drive},  Now: {drive}')
                             build_receive_plot('normal', drive)
             else:
@@ -927,14 +928,18 @@ def update_receive_plot():
                         log.debug(f'{receive_script} not found. Building it now...')
                         build_receive_plot('portable', drive)
                     else:
-                        if chianas.current_plot_replacement_drive == drive:
-                            log.debug(f'Currently Configured Replacement Drive: {chianas.current_plot_replacement_drive}')
-                            log.debug(f'System Selected Replacement Drive:      {drive}')
-                            log.debug('Configured and Selected Drives Match!')
-                            log.debug(f'No changes necessary to {receive_script}')
+                        if remote_transfer_active:
+                            log.debug('Remote Transfer in Progress, will try again soon!')
+                            quit()  # TODO Think about what we really want to do here. If we are running a remote transfer, can we still do other things?
                         else:
-                            notify('Plot Replacement Drive Updated', f'Plot Drive Updated: Was: {chianas.current_plot_replacement_drive},  Now: {drive}')
-                            build_receive_plot('portable', drive)
+                            if chianas.current_plot_replacement_drive == drive:
+                                log.debug(f'Currently Configured Replacement Drive: {chianas.current_plot_replacement_drive}')
+                                log.debug(f'System Selected Replacement Drive:      {drive}')
+                                log.debug('Configured and Selected Drives Match!')
+                                log.debug(f'No changes necessary to {receive_script}')
+                            else:
+                                notify('Plot Replacement Drive Updated', f'Plot Drive Updated: Was: {chianas.current_plot_replacement_drive},  Now: {drive}')
+                                build_receive_plot('portable', drive)
                 else:
                     log.debug(f'ERROR: Replace Plots Configured, but no old plots exist!')
                     quit()
@@ -948,18 +953,47 @@ def update_receive_plot():
                     log.debug(f'{receive_script} not found. Building it now...')
                     build_receive_plot('portable', drive)
                 else:
-                    if chianas.current_plot_replacement_drive == drive:
-                        log.debug(f'Currently Configured Replacement Drive: {chianas.current_plot_replacement_drive}')
-                        log.debug(f'System Selected Replacement Drive:      {drive}')
-                        log.debug('Configured and Selected Drives Match!')
-                        log.debug(f'No changes necessary to {receive_script}')
+                    if remote_transfer_active:
+                        log.debug('Remote Transfer in Progress, will try again soon!')
+                        quit()  # TODO Think about what we really want to do here. If we are running a remote transfer, can we still do other things?
                     else:
-                       # send_new_plot_disk_email()  # This is the full Plot drive report. This is in addition to the generic email sent by the
-                                                    # notify() function. - TODO Do we need to send this here or do we need to update the function?
-                        notify('Plot Replacement Drive Updated', f'Plot Drive Updated: Was: {chianas.current_plot_replacement_drive},  Now: {drive}')
-                        build_receive_plot('portable', drive)
+                        if chianas.current_plot_replacement_drive == drive:
+                            log.debug(f'Currently Configured Replacement Drive: {chianas.current_plot_replacement_drive}')
+                            log.debug(f'System Selected Replacement Drive:      {drive}')
+                            log.debug('Configured and Selected Drives Match!')
+                            log.debug(f'No changes necessary to {receive_script}')
+                        else:
+                            notify('Plot Replacement Drive Updated', f'Plot Drive Updated: Was: {chianas.current_plot_replacement_drive},  Now: {drive}')
+                            build_receive_plot('portable', drive)
             else:
-                log.debug(f'ERROR: Replace Plots Configured, but no old plots exist!')
+                log.debug(f'ERROR: Replace Plots Configured, but no old plots exist! Defaulting to filling our empty drives...')
+                if (get_all_available_system_space("free")[1]) > chianas.empty_drives_low_water_mark:
+                    log.debug('Found Empty Drive Space!')
+                    log.debug(f'Low Water Mark: {chianas.empty_drives_low_water_mark} and we have {get_all_available_system_space("free")[1]} available')
+                    drive = get_plot_drive_to_use()[0]
+                    if not os.path.isfile(receive_script):
+                        log.debug(f'{receive_script} not found. Building it now...')
+                        build_receive_plot('normal', drive)
+                    else:
+                        if remote_transfer_active:
+                            log.debug('Remote Transfer in Progress, will try again soon!')
+                            quit()  # TODO Think about what we really want to do here. If we are running a remote transfer, can we still do other things?
+                        else:
+                            if chianas.current_plotting_drive == drive:
+                                log.debug(f'Currently Configured Plot Drive: {chianas.current_plotting_drive}')
+                                log.debug(f'System Selected Plot Drive:      {drive}')
+                                log.debug('Configured and Selected Drives Match!')
+                                log.debug(f'No changes necessary to {receive_script}')
+                                log.debug(
+                                    f'Plots left available on configured plotting drive: {get_drive_info("space_free_plots_by_mountpoint", chianas.current_plotting_drive)}')
+                            else:
+                                send_new_plot_disk_email()  # This is the full Plot drive report. This is in addition to the generic email sent by the notify() function.
+                                notify('Plot Drive Updated',
+                                       f'Plot Drive Updated: Was: {chianas.current_plotting_drive},  Now: {drive}')
+                                build_receive_plot('normal', drive)
+                else:
+                    log.debug('We have a problem. There are no old plots to replace and we have run out of empty drive space, not sure what to do now but phone home.....')
+                    notify('No Old Plots & No Drive Space Available', 'We have a problem. There are no old plots to replace and we have run out of empty drive space, I need your help!')
                 quit()
 
 def build_receive_plot(type, drive):
@@ -982,6 +1016,57 @@ def build_receive_plot(type, drive):
         log.info(f'Was: {chianas.current_plotting_drive},  Now: {drive}')
 
 
+def check_for_active_remote_transfer():
+    """
+    Function to check and verify if we have a remote transfer active to prevent
+    overloading drives during local and remote copies of plots.
+    """
+    log.debug('check_for_active_remote_transfer() called')
+    if os.path.isfile(remote_transfer_active_file):
+        log.debug(f'A Remote Transfer appears to be in Progress, checking for network traffic on interface: {chianas.plot_receive_interface}.')
+        if check_network_activity():
+            log.debug('Network traffic has been detected, a Remote Transfer is in progress.')
+            return True
+        else:
+            log.debug(f'Remote Transfer file present but there is no network traffic on interface: {chianas.plot_receive_interface}')
+            log.debug('Resetting Remote Transfer file now......')
+            os.remove(remote_transfer_active_file)
+            return False
+    else:
+        log.debug('Remote Transfer File does not exist, lets check for network traffic to verify....')
+        if check_network_activity():
+            log.debug('Network traffic has been detected, a Remote Transfer is in progress.')
+            return True
+        else:
+            log.debug('No Current Remote Transfers are taking place.')
+            return False
+
+def check_network_activity():
+    """
+    Here we are checking network activity on the network interface we are receiving plots on from our plotter. If there is
+    network activity, then we are most likely receiving a plot and don't want to make any changes.
+    """
+    log.debug('check_network_activity() called')
+    try:
+        subprocess.call([network_check, chianas.plot_receive_interface])
+    except subprocess.CalledProcessError as e:
+        log.warning(e.output)
+    with open(network_check_output, 'rb') as f:
+        f.seek(-2, os.SEEK_END)
+        while f.read(1) != b'\n':
+            f.seek(-2, os.SEEK_CUR)
+        last_line = f.readline().decode()
+        network_traffic_load = float((str.split(last_line)[9]))
+    if network_traffic_load >= chianas.plot_receive_interface_threshold:
+        log.debug(f'Network Activity detected on {chianas.plot_receive_interface}')
+        os.remove(network_check_output)
+        return True
+    else:
+        log.debug(f'No Network Activity detected on {chianas.plot_receive_interface}')
+        os.remove(network_check_output)
+        return False
+
+
 def send_new_plot_disk_email():
     """
     This is the function that we call when we want to send an email letting you know that a new
@@ -1580,7 +1665,6 @@ def main():
         send_new_plot_notification()
         update_receive_plot()
 
-
-
 if __name__ == '__main__':
     main()
+