I am running PollingObserverVFS on a directory that is on Dell ISILON Storage that has over 2 million files and it takes around 2 hours 30 min to detect changes. Is there a way to improve performance to detect changes faster like parallel processing or any other settings changes.
The code is pooling dell isilon storage to detect folder changes.
import datetime
import time
import os
from watchdog.observers.polling import PollingObserverVFS
from watchdog.events import FileSystemEventHandler
if __name__ == "__main__":
# Set format for displaying path
my_event_handler = FileSystemEventHandler()
path = "\\\\dd\\ARCHIVE\\"
print(path)
# Initialize logging event handler
#event_handler = LoggingEventHandler()
def on_any_event(event):
print(datetime.datetime.now())
if event.is_directory:
if event.event_type == 'created':
print("Watchdog received directory create event - % s." % event.src_path)
elif event.event_type == 'modified':
# Event is modified, you can process it now
print("Watchdog received directory modified event - % s." % event.src_path)
elif event.event_type == 'moved':
# Event is modified, you can process it now
print("Watchdog received directory moved event - % s." % event.src_path)
print("Moved or Renamed directory path - % s." %event.dest_path )
elif event.event_type == 'deleted':
# Event is modified, you can process it now
print("Watchdog received deleted event - % s." % event.src_path)
elif event.event_type == 'created':
# Event is modified, you can process it now
print("Watchdog received file created event - % s." % event.src_path)
elif event.event_type == 'modified':
# Event is modified, you can process it now
print("Watchdog received modified file event - % s." % event.src_path)
elif event.event_type == 'moved':
# Event is modified, you can process it now
print("Watchdog received moved file event - % s." % event.src_path)
print("Moved or Renamed path - % s." %event.dest_path )
elif event.event_type == 'deleted':
# Event is modified, you can process it now
print("Watchdog received deleted file event - % s." % event.src_path)
my_event_handler.on_any_event = on_any_event
observer = PollingObserverVFS(stat=os.stat, listdir=os.listdir,polling_interval=60)
observer.schedule(my_event_handler, path, recursive=True)
# Start the observer
observer.start()
try:
while True:
# Set the thread sleep time
time.sleep(60)
except KeyboardInterrupt:
observer.stop()
observer.join()