I'm trying to build a big video from ~400 videos, however I'm facing issues with "Too many files open". I've read the documentation (and saw this post).
However I'm still getting the same error: OSError: [Errno 24] Too many open files
I've investigated and don't know what else to do if I'm already deleting each opened movie with the method close_files.
The code:
from moviepy.editor import *
import json
import os
import urllib
import shutil
import re
import api.constants
import traceback
from api.constants import *
import datetime
import string
def close_files(files):
for file in files:
try:
if not (isinstance(file,ImageClip) or isinstance(file,TextClip)):
del file.reader
del file
except Exception,e:
print "Exception: "+str(e)
def max_text(title,author):
return title
#if len(title) > 120:
# return title[0:(120- len(author))]+"..."
#else:
# return title
def clean(text):
return ''.join(filter(lambda x: x in string.printable, text))
def compile_videos(fileName,serverjson,post_to):
try:
now = datetime.datetime.now()
print "Starting at: "+str(now)
#Video's dimensions
w = 1440
h = 1440
videoDimension = w,h
#Video offset
videoOffset = 580
titleOffset = 0
#Movie dimensions
w2 = 2560
h2 = 1440
movieDimensions = w2,h2
#Title dimensions
w3 = 540
h3 = 1440
titleDimensions = w3,h3
#Left Holder Dimension
holderDimensions = 500,500
holderPosition = 50,450
#Holder
left_holder = ImageClip("assets/everyday_icon.png").set_position(holderPosition)
if post_to=="7min":
left_holder = ImageClip("assets/7min_icon.png").set_position(holderPosition)
elif post_to=="7x7":
left_holder = ImageClip("assets/7x7_icon.png").set_position(holderPosition)
#Temporal directory
directory = 'tmp'
if os.path.exists(directory):
shutil.rmtree(directory)
if not os.path.exists(directory):
os.makedirs(directory)
#Read Videos/Files
data = serverjson
videos_and_subtitles = []
#We add the holder so that there's always a background
videos_and_subtitles.append(left_holder)
currentDuration = 0
videoId = ""
preprocessedVideos = []
counter = 0
videoNumber = 0
print "Total videos: "+str(len(data["videos"]))
for videoObject in data["videos"]:
counter+=1
#Download video and thumbnail
downloader=urllib.URLopener()
videoId = videoObject[API_VIDEO_ID]
videoPath = directory+"/"+str(videoObject[API_VIDEO_ID])+'_video.mp4'
thumbPath = directory+"/"+str(videoObject[API_VIDEO_ID])+'_thumb.jpg'
try:
downloader.retrieve(str(videoObject[API_VIDEO_URL]),videoPath)
downloader.retrieve(str(videoObject[API_THUMB_URL]),thumbPath)
except Exception,e:
print "Exception: "+str(e)
print "Video ID: "+str(videoId)
traceback.print_exc()
continue
#Create new video and update video duration's offset
newVideo = VideoFileClip(videoPath).resize(height=w,width=h).set_position((videoOffset,titleOffset)).set_start(currentDuration)
#Append new video to videos
videos_and_subtitles.append(newVideo)
#Append subtitle to Subtitles
videoName = clean(videoObject[API_NAME])
videoAuthor = clean(videoObject[API_AUTHOR])
newSubtitleText = clean(max_text(videoName,videoAuthor)+" \n\n"+videoObject[API_AUTHOR])
newSubtitle = ( TextClip(newSubtitleText,fontsize=70,color='white',font='Helvetica-Narrow',align='center',method='caption',size=titleDimensions).set_start(currentDuration).set_position((videoOffset+w,0)).set_duration(newVideo.duration) )
videos_and_subtitles.append(newSubtitle)
currentDuration+=newVideo.duration
#Preprocess videos
if counter%50==0 or len(data["videos"])==(counter):
currentFilename=directory+"/"+str(videoNumber)+fileName
result = CompositeVideoClip(videos_and_subtitles,size=movieDimensions,bg_color=(0,164,119)).set_duration(currentDuration).write_videofile(filename=currentFilename,preset='ultrafast',fps=24)
preprocessedVideos.append(VideoFileClip(currentFilename))
close_files(videos_and_subtitles)
videos_and_subtitles = []
currentDuration = 0
videoNumber+=1
if (videoObject==data["videos"][-1]):
break
print "Next video"
print "Compiling video"
result = concatenate_videoclips(preprocessedVideos).write_videofile(filename=directory+"/"+fileName,preset='ultrafast')
#result = CompositeVideoClip(videos_and_subtitles,size=movieDimensions,bg_color=(0,164,119)).set_duration(currentDuration).write_videofile(filename=directory+"/"+fileName,preset='ultrafast')
print "Video Compiled"
now = datetime.datetime.now()
print "Finished at: "+str(now)
return fileName
except Exception,e:
print "Exception: "+str(e)
print "Video ID: "+str(videoId)
traceback.print_exc()
return None
I'm processing the files in batches of 50 because it takes longer if I do it in smaller batches. The problem isn't having 50 files open at once. The problem happens after the first or second batch (~100-150 files processed), this points to the files not being closed (otherwise the first batch wouldn't have been processed correctly). This means the same happens if I do it in batches of 10, it's just that the files are not being closed correctly
Other posts like this one, as you can see the answer:
Your test script overwrites f each iteration, which means that the file will get closed each time. Both logging to files and subprocess with pipes use up descriptors, which can lead to exhaustion.
Doesn't quite apply, as I'm overwriting the instance every 50 iterations, however I'm still having the problem.
Also, I can't call "close" on a VideoFileClip.
