Исправлены ошибки, доработан класс opdsDaemon. Запущена проверка работы демона. (84e7f1d7) · Commits · Mirror / Simple OPDS

conf/sopds.conf

+1 −1

Original line number	Diff line number	Diff line
		@@ -122,7 +122,7 @@ book_shelf = yes
		# Параметр logfile задает имя файла, куда будут складываться логи (путь задавать не нужно - логи будут находится в папке logs)
		# По умолчанию logfile = sopds.log
		# loglevel может быть одним из следующих значений: debug, info, warning, error, critical, none.
		# По умолчанию loglevel=none
		# По умолчанию loglevel=info
		logfile=sopds.log
		loglevel=info

logs/sopds.log

+81 −0

Original line number	Diff line number	Diff line
		@@ -66,3 +66,84 @@ mysql.connector.errors.DatabaseError: 1205 (HY000): Lock wait timeout exceeded;
		2014-04-20 21:51:13,086 INFO Bad archives : 0
		2014-04-20 21:51:13,086 INFO Time estimated:0 hours, 7 minutes, 31 seconds.
		2014-04-20 21:52:11,941 INFO ***** Starting sopds-scan...
		2014-04-20 21:57:33,098 INFO Books added : 0
		2014-04-20 21:57:33,099 INFO Books skipped : 5569
		2014-04-20 21:57:33,099 INFO Books DB entries deleted : 0
		2014-04-20 21:57:33,100 INFO Books in archives: 0
		2014-04-20 21:57:33,100 INFO Archives scanned : 35
		2014-04-20 21:57:33,100 INFO Archives skipped : 127
		2014-04-20 21:57:33,101 INFO Bad archives : 0
		2014-04-20 21:57:33,101 INFO Time estimated:0 hours, 5 minutes, 23 seconds.
		2014-04-20 22:39:03,119 INFO ***** Starting sopds-scan...
		2014-04-20 22:42:45,399 INFO Books added : 0
		2014-04-20 22:42:45,400 INFO Books skipped : 5569
		2014-04-20 22:42:45,400 INFO Books DB entries deleted : 0
		2014-04-20 22:42:45,400 INFO Books in archives: 0
		2014-04-20 22:42:45,401 INFO Archives scanned : 35
		2014-04-20 22:42:45,401 INFO Archives skipped : 127
		2014-04-20 22:42:45,401 INFO Bad archives : 0
		2014-04-20 22:42:45,402 INFO Time estimated:0 hours, 3 minutes, 42 seconds.
		2014-04-20 22:51:37,850 INFO ***** Starting sopds-scan...
		2014-04-20 22:55:05,323 INFO Books added : 0
		2014-04-20 22:55:05,324 INFO Books skipped : 5569
		2014-04-20 22:55:05,324 INFO Books DB entries deleted : 0
		2014-04-20 22:55:05,324 INFO Books in archives: 0
		2014-04-20 22:55:05,325 INFO Archives scanned : 35
		2014-04-20 22:55:05,325 INFO Archives skipped : 127
		2014-04-20 22:55:05,325 INFO Bad archives : 0
		2014-04-20 22:55:05,326 INFO Time estimated:0 hours, 3 minutes, 27 seconds.
		2014-04-20 23:06:31,166 INFO sopdsDaemon initializing...
		2014-04-20 23:06:31,167 INFO sopdsDaemon stopping...
		2014-04-20 23:06:56,923 INFO sopdsDaemon initializing...
		2014-04-20 23:06:56,924 INFO sopdsDaemon starting...
		2014-04-20 23:06:56,975 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:06:56,976 INFO ***** Starting sopds-scan...
		2014-04-20 23:07:34,128 INFO sopdsDaemon initializing...
		2014-04-20 23:10:13,182 INFO Books added : 0
		2014-04-20 23:10:13,182 INFO Books skipped : 5569
		2014-04-20 23:10:13,182 INFO Books DB entries deleted : 0
		2014-04-20 23:10:13,183 INFO Books in archives: 0
		2014-04-20 23:10:13,183 INFO Archives scanned : 35
		2014-04-20 23:10:13,183 INFO Archives skipped : 127
		2014-04-20 23:10:13,184 INFO Bad archives : 0
		2014-04-20 23:10:13,184 INFO Time estimated:0 hours, 3 minutes, 16 seconds.
		2014-04-20 23:10:17,668 INFO sopdsDaemon initializing...
		2014-04-20 23:10:17,669 INFO sopdsDaemon status checking...
		2014-04-20 23:10:26,749 INFO sopdsDaemon initializing...
		2014-04-20 23:10:28,336 INFO sopdsDaemon stopping...
		2014-04-20 23:11:45,977 INFO sopdsDaemon initializing...
		2014-04-20 23:11:45,978 INFO sopdsDaemon starting...
		2014-04-20 23:11:46,034 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:11:46,035 INFO ***** Starting sopds-scan...
		2014-04-20 23:12:24,175 INFO sopdsDaemon initializing...
		2014-04-20 23:12:24,177 INFO sopdsDaemon starting...
		2014-04-20 23:12:52,345 INFO sopdsDaemon initializing...
		2014-04-20 23:12:52,346 INFO sopdsDaemon starting...
		2014-04-20 23:12:52,421 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:12:52,422 INFO ***** Starting sopds-scan...
		2014-04-20 23:13:42,993 INFO sopdsDaemon initializing...
		2014-04-20 23:13:42,994 INFO sopdsDaemon starting...
		2014-04-20 23:13:43,081 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:13:43,083 INFO ***** Starting sopds-scan...
		2014-04-20 23:16:55,462 INFO Books added : 0
		2014-04-20 23:16:55,463 INFO Books skipped : 5569
		2014-04-20 23:16:55,463 INFO Books DB entries deleted : 0
		2014-04-20 23:16:55,463 INFO Books in archives: 0
		2014-04-20 23:16:55,464 INFO Archives scanned : 35
		2014-04-20 23:16:55,464 INFO Archives skipped : 127
		2014-04-20 23:16:55,464 INFO Bad archives : 0
		2014-04-20 23:16:55,465 INFO Time estimated:0 hours, 3 minutes, 12 seconds.
		2014-04-20 23:18:04,858 INFO sopdsDaemon initializing...
		2014-04-20 23:18:04,859 INFO sopdsDaemon status checking...
		2014-04-20 23:18:17,781 INFO sopdsDaemon initializing...
		2014-04-20 23:18:17,782 INFO sopdsDaemon stopping...
		2014-04-20 23:18:52,474 INFO sopdsDaemon initializing...
		2014-04-20 23:18:52,475 INFO sopdsDaemon starting...
		2014-04-20 23:18:52,565 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:18:52,570 INFO ***** Starting sopds-scan...
		2014-04-20 23:21:17,320 INFO sopdsDaemon initializing...
		2014-04-20 23:21:17,321 INFO sopdsDaemon stopping...
		2014-04-20 23:21:39,494 INFO sopdsDaemon initializing...
		2014-04-20 23:21:39,495 INFO sopdsDaemon starting...
		2014-04-20 23:21:39,545 INFO sopdsDaemon entering in main loop...
		2014-04-20 23:21:39,545 INFO ***** Starting sopds-scan...

py/pycache/sopdscfg.cpython-33.pyc

(6.76 KiB)

File changed.

No diff preview for this file type.

View original file

View changed file

py/sopds-scan.py

+20 −1

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		# -- coding: utf-8 --

		import logging
		import sopdscfg
		from sopdscan import opdsScanner
		from optparse import OptionParser
		@@ -17,7 +18,25 @@ if (__name__=="__main__"):
		if CFG_FILE=='': cfg=sopdscfg.cfgreader()
		else: cfg=sopdscfg.cfgreader(CFG_FILE)

		scanner=opdsScanner(cfg,VERBOSE)
		logger = logging.getLogger('')
		logger.setLevel(cfg.LOGLEVEL)
		formatter=logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')

		if cfg.LOGLEVEL!=logging.NOTSET:
		# Создаем обработчик для записи логов в файл
		fh = logging.FileHandler(cfg.LOGFILE)
		fh.setLevel(cfg.LOGLEVEL)
		fh.setFormatter(formatter)
		logger.addHandler(fh)

		if VERBOSE:
		# Создадим обработчик для вывода логов на экран с максимальным уровнем вывода
		ch = logging.StreamHandler()
		ch.setLevel(logging.DEBUG)
		ch.setFormatter(formatter)
		logger.addHandler(ch)

		scanner=opdsScanner(cfg,logger)
		scanner.log_options()
		scanner.scan_all()
		scanner.log_stats()

py/sopdscan.py

+43 −44

Original line number	Diff line number	Diff line
		@@ -12,33 +12,32 @@ import zipf
		import logging

		class opdsScanner:
		def __init__(self, cfg, verbose=False):
		self.VERBOSE=verbose
		def __init__(self, cfg, logger):
		self.cfg=cfg
		self.logger=logger
		self.opdsdb=None
		self.fb2parser=None
		self.init_stats()
		self.init_logger()
		self.init_parser()
		zipf.ZIP_CODEPAGE=self.cfg.ZIP_CODEPAGE
		self.extensions_set={x for x in self.cfg.EXT_LIST}

		def init_logger(self):
		if self.cfg.LOGLEVEL!=logging.NOTSET:
		# Создаем обработчик для записи логов в файл
		self.fh = logging.FileHandler(self.cfg.LOGFILE)
		self.fh.setLevel(self.cfg.LOGLEVEL)

		if self.VERBOSE:
		# Создадим обработчик для вывода логов на экран с максимальным уровнем вывода
		self.ch = logging.StreamHandler()
		self.ch.setLevel(logging.DEBUG)

		logformat='%(asctime)s %(levelname)-8s %(message)s'
		if self.VERBOSE:
		logging.basicConfig(format = logformat, level = logging.DEBUG, handlers=(self.fh,self.ch))
		else:
		logging.basicConfig(format = logformat, level = logging.INFO, handlers=(self.fh,))
		# def init_logger(self):
		# if self.cfg.LOGLEVEL!=logging.NOTSET:
		# # Создаем обработчик для записи логов в файл
		# self.fh = logging.FileHandler(self.cfg.LOGFILE)
		# self.fh.setLevel(self.cfg.LOGLEVEL)
		#
		# if self.VERBOSE:
		# # Создадим обработчик для вывода логов на экран с максимальным уровнем вывода
		# self.ch = logging.StreamHandler()
		# self.ch.setLevel(logging.DEBUG)
		#
		# logformat='%(asctime)s %(levelname)-8s %(message)s'
		# if self.VERBOSE:
		# logging.basicConfig(format = logformat, level = logging.DEBUG, handlers=(self.fh,self.ch))
		# else:
		# logging.basicConfig(format = logformat, level = logging.INFO, handlers=(self.fh,))

		def init_stats(self):
		self.t1=datetime.timedelta(seconds=time.time())
		@@ -55,32 +54,32 @@ class opdsScanner:
		self.fb2parser=sopdsparse.fb2parser(self.cfg.COVER_EXTRACT)

		def log_options(self):
		logging.info(' ***** Starting sopds-scan...')
		logging.debug('OPTIONS SET')
		if self.cfg.CONFIGFILE!=None: logging.debug('configfile = '+self.cfg.CONFIGFILE)
		if self.cfg.ROOT_LIB!=None: logging.debug('root_lib = '+self.cfg.ROOT_LIB)
		if self.cfg.FB2TOEPUB_PATH!=None: logging.debug('fb2toepub = '+self.cfg.FB2TOEPUB_PATH)
		if self.cfg.FB2TOMOBI_PATH!=None: logging.debug('fb2tomobi = '+self.cfg.FB2TOMOBI_PATH)
		if self.cfg.TEMP_DIR!=None: logging.debug('temp_dir = '+self.cfg.TEMP_DIR)
		self.logger.info(' ***** Starting sopds-scan...')
		self.logger.debug('OPTIONS SET')
		if self.cfg.CONFIGFILE!=None: self.logger.debug('configfile = '+self.cfg.CONFIGFILE)
		if self.cfg.ROOT_LIB!=None: self.logger.debug('root_lib = '+self.cfg.ROOT_LIB)
		if self.cfg.FB2TOEPUB_PATH!=None: self.logger.debug('fb2toepub = '+self.cfg.FB2TOEPUB_PATH)
		if self.cfg.FB2TOMOBI_PATH!=None: self.logger.debug('fb2tomobi = '+self.cfg.FB2TOMOBI_PATH)
		if self.cfg.TEMP_DIR!=None: self.logger.debug('temp_dir = '+self.cfg.TEMP_DIR)

		def log_stats(self):
		self.t2=datetime.timedelta(seconds=time.time())
		logging.info('Books added : '+str(self.books_added))
		logging.info('Books skipped : '+str(self.books_skipped))
		self.logger.info('Books added : '+str(self.books_added))
		self.logger.info('Books skipped : '+str(self.books_skipped))
		if self.cfg.DELETE_LOGICAL:
		logging.info('Books deleted : '+str(self.books_deleted))
		self.logger.info('Books deleted : '+str(self.books_deleted))
		else:
		logging.info('Books DB entries deleted : '+str(self.books_deleted))
		logging.info('Books in archives: '+str(self.books_in_archives))
		logging.info('Archives scanned : '+str(self.arch_scanned))
		logging.info('Archives skipped : '+str(self.arch_skipped))
		logging.info('Bad archives : '+str(self.bad_archives))
		self.logger.info('Books DB entries deleted : '+str(self.books_deleted))
		self.logger.info('Books in archives: '+str(self.books_in_archives))
		self.logger.info('Archives scanned : '+str(self.arch_scanned))
		self.logger.info('Archives skipped : '+str(self.arch_skipped))
		self.logger.info('Bad archives : '+str(self.bad_archives))

		t=self.t2-self.t1
		seconds=t.seconds%60
		minutes=((t.seconds-seconds)//60)%60
		hours=t.seconds//3600
		logging.info('Time estimated:'+str(hours)+' hours, '+str(minutes)+' minutes, '+str(seconds)+' seconds.')
		self.logger.info('Time estimated:'+str(hours)+' hours, '+str(minutes)+' minutes, '+str(seconds)+' seconds.')

		def scan_all(self):
		self.opdsdb=sopdsdb.opdsDatabase(self.cfg.DB_NAME,self.cfg.DB_USER,self.cfg.DB_PASS,self.cfg.DB_HOST,self.cfg.ROOT_LIB)
		@@ -120,25 +119,25 @@ class opdsScanner:
		filelist = z.namelist()
		for n in filelist:
		try:
		logging.debug('Start process ZIP file = '+file+' book file = '+n)
		self.logger.debug('Start process ZIP file = '+file+' book file = '+n)
		file_size=z.getinfo(n).file_size
		self.processfile(n,file,z.open(n),1,file_size,cat_id=cat_id)
		except:
		logging.error('Error processing ZIP file = '+file+' book file = '+n)
		self.logger.error('Error processing ZIP file = '+file+' book file = '+n)
		z.close()
		self.arch_scanned+=1
		except:
		logging.error('Error while read ZIP archive. File '+file+' corrupt.')
		self.logger.error('Error while read ZIP archive. File '+file+' corrupt.')
		self.bad_archives+=1
		else:
		self.arch_skipped+=1
		logging.debug('Skip ZIP archive '+rel_file+'. Already scanned.')
		self.logger.debug('Skip ZIP archive '+rel_file+'. Already scanned.')

		def processfile(self,name,full_path,file,archive=0,file_size=0,cat_id=0):
		(n,e)=os.path.splitext(name)
		if e.lower() in self.extensions_set:
		rel_path=os.path.relpath(full_path,self.cfg.ROOT_LIB)
		logging.debug("Attempt to add book "+rel_path+"/"+name)
		self.logger.debug("Attempt to add book "+rel_path+"/"+name)
		self.fb2parser.reset()
		if self.opdsdb.findbook(name,rel_path,1)==0:
		if archive==0:
		@@ -166,7 +165,7 @@ class opdsScanner:
		docdate=self.fb2parser.docdate.getvalue()[0].strip();

		if self.fb2parser.parse_error!=0:
		logging.warning(rel_path+' - '+name+' fb2 parse warning ['+self.fb2parser.parse_errormsg+']')
		self.logger.warning(rel_path+' - '+name+' fb2 parse warning ['+self.fb2parser.parse_errormsg+']')

		if title=='': title=n

		@@ -177,11 +176,11 @@ class opdsScanner:
		try:
		create_cover(book_id)
		except:
		logging.error('Error extract cover from file '+name)
		self.logger.error('Error extract cover from file '+name)

		if archive==1:
		self.books_in_archives+=1
		logging.debug("Book "+rel_path+"/"+name+" Added ok.")
		self.logger.debug("Book "+rel_path+"/"+name+" Added ok.")

		idx=0
		for l in self.fb2parser.author_last.getvalue():
		@@ -198,7 +197,7 @@ class opdsScanner:

		else:
		self.books_skipped+=1
		logging.debug("Book "+rel_path+"/"+name+" Already in DB.")
		self.logger.debug("Book "+rel_path+"/"+name+" Already in DB.")

		def create_cover(self,book_id):
		ictype=self.fb2parser.cover_image.getattr('content-type')