Commit 0dbcc0b5 authored by Dmitry Shelepnev's avatar Dmitry Shelepnev
Browse files

Optimize dDB Field Sizes and indexes

parent fb806868

opds_catalog/feeds.py

+1 −1
Original line number Diff line number Diff line
@@ -238,7 +238,7 @@ class CatalogsFeed(AuthFeed): 


    def items(self, obj):

        cat, current_page = obj

        catalogs_list = Catalog.objects.filter(parent=cat).order_by("cat_name", "cat_type")

        catalogs_list = Catalog.objects.filter(parent=cat).order_by("cat_name")

        # prefetch_related on sqlite on items >999 therow error "too many SQL variables"

        #books_list = Book.objects.filter(catalog=cat).prefetch_related('authors','genres','series').order_by("title")

        books_list = Book.objects.filter(catalog=cat).order_by("search_title")

opds_catalog/models.py

+43 −28
Original line number Diff line number Diff line
@@ -10,47 +10,62 @@ counter_allauthors = 'allauthors' 
counter_allgenres = 'allgenres'

counter_allseries = 'allseries'



SIZE_BOOK_FILENAME   = 512

SIZE_BOOK_PATH       = 512

SIZE_BOOK_FORMAT     = 8

SIZE_BOOK_DOCDATE    = 32

SIZE_BOOK_LANG       = 16

SIZE_BOOK_TITLE      = 512

SIZE_BOOK_ANNOTATION = 10000



SIZE_CAT_CATNAME     = 190

SIZE_CAT_PATH        = SIZE_BOOK_PATH



SIZE_AUTHOR_NAME     = 128



SIZE_GENRE           = 32

SIZE_GENRE_SECTION   = 64

SIZE_GENRE_SUBSECTION = 100



SIZE_SERIES          = 150





LangCodes = {1:'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЬЫЪЭЮЯабвгдеёжзийклмнопрстуфхцчшщьыъэюя',

             2:'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz',

             3:'0123456789'}

lang_menu = {1:_('Cyrillic'), 2:_('Latin'), 3:_('Digits'), 9:_('Other symbols'), 0:_('Show all')}



class Book(models.Model):

    filename = models.CharField(db_index=True, max_length=256)

    path = models.CharField(db_index=True, max_length=1000)

    filesize = models.IntegerField(null=False, default=0, db_index=True)

    format = models.CharField(max_length=8, db_index=True)

    filename = models.CharField(max_length=SIZE_BOOK_FILENAME)

    path = models.CharField(max_length=SIZE_BOOK_PATH)

    filesize = models.IntegerField(null=False, default=0)

    format = models.CharField(max_length=SIZE_BOOK_FORMAT)

    catalog = models.ForeignKey('Catalog',db_index=True)

    cat_type = models.IntegerField(null=False, default=0)

    registerdate = models.DateTimeField(db_index=True, null=False, default=timezone.now)

    docdate = models.CharField(max_length=32)

    registerdate = models.DateTimeField(null=False, default=timezone.now)

    docdate = models.CharField(max_length=SIZE_BOOK_DOCDATE,db_index=True)

    #favorite = models.IntegerField(null=False, default=0)

    lang = models.CharField(max_length=16)

    title = models.CharField(max_length=256, db_index=True)

    search_title = models.CharField(max_length=256, default=None, db_index=True)

    annotation = models.CharField(max_length=10000)

    lang_code = models.IntegerField(db_index=True, null=False, default=9)

    lang = models.CharField(max_length=SIZE_BOOK_LANG)

    title = models.CharField(max_length=SIZE_BOOK_TITLE, db_index=True)

    search_title = models.CharField(max_length=SIZE_BOOK_TITLE, default=None, db_index=True)

    annotation = models.CharField(max_length=SIZE_BOOK_ANNOTATION)

    lang_code = models.IntegerField(null=False, default=9, db_index=True)

    avail = models.IntegerField(null=False, default=0, db_index=True)

    authors = models.ManyToManyField('Author', through='bauthor')

    genres = models.ManyToManyField('Genre', through='bgenre')

    series = models.ManyToManyField('Series', through='bseries')



#    class Meta:

#        index_together = [

#            ["title", "format", "filesize"]

#        ]



class Catalog(models.Model):

    parent = models.ForeignKey('self', null=True, db_index=True)

    cat_name = models.CharField(db_index=True, max_length=128)

    path = models.CharField(db_index=True, max_length=1000)

    cat_name = models.CharField(max_length=SIZE_CAT_CATNAME, db_index=True)

    path = models.CharField(max_length=SIZE_CAT_PATH, db_index=True)

    cat_type = models.IntegerField(null=False, default=0)

    cat_size = models.BigIntegerField(null=True, default=0)



class Author(models.Model):

    full_name = models.CharField(db_index=True, max_length=128, default=None)

    search_full_name = models.CharField(db_index=True, max_length=128, default=None)

    lang_code = models.IntegerField(db_index=True, null=False, default=9)

    full_name = models.CharField(max_length=SIZE_AUTHOR_NAME, default=None, db_index=True)

    search_full_name = models.CharField(max_length=SIZE_AUTHOR_NAME, default=None, db_index=True)

    lang_code = models.IntegerField(null=False, default=9, db_index=True)





class bauthor(models.Model):
@@ -62,9 +77,9 @@ class bauthor(models.Model): 
#        ]



class Genre(models.Model):

    genre = models.CharField(db_index=True, max_length=32)

    section = models.CharField(db_index=True, max_length=64)

    subsection = models.CharField(db_index=True, max_length=100)

    genre = models.CharField(max_length=SIZE_GENRE, db_index=True)

    section = models.CharField(max_length=SIZE_GENRE_SECTION, db_index=True)

    subsection = models.CharField(max_length=SIZE_GENRE_SUBSECTION, db_index=True)



class bgenre(models.Model):

    book = models.ForeignKey('Book', db_index=True)
@@ -75,9 +90,9 @@ class bgenre(models.Model): 
#        ]



class Series(models.Model):

    ser = models.CharField(db_index=True, max_length=80)

    search_ser = models.CharField(db_index=True, max_length=80, default=None)

    lang_code = models.IntegerField(db_index=True, null=False, default=9)

    ser = models.CharField(max_length=80, db_index=True)

    search_ser = models.CharField(max_length=SIZE_SERIES, default=None, db_index=True)

    lang_code = models.IntegerField(null=False, default=9,db_index=True)



class bseries(models.Model):

    book = models.ForeignKey('Book', db_index=True)
@@ -91,7 +106,7 @@ class bseries(models.Model): 
class bookshelf(models.Model):

    user = models.ForeignKey(User, db_index=True)

    book = models.ForeignKey(Book, db_index=True)

    readtime = models.DateTimeField(null=False, default=timezone.now)

    readtime = models.DateTimeField(null=False, default=timezone.now, db_index=True)





class CounterManager(models.Manager):

opds_catalog/opdsdb.py

+13 −28
Original line number Diff line number Diff line
@@ -4,6 +4,8 @@ from django.db.models import Q 
from django.utils.translation import ugettext as _



from opds_catalog.models import Book, Catalog, Author, Genre, Series, bseries, bauthor, bgenre, bookshelf, Counter, LangCodes

from opds_catalog.models import SIZE_BOOK_FILENAME, SIZE_BOOK_PATH, SIZE_BOOK_FORMAT, SIZE_BOOK_DOCDATE, SIZE_BOOK_LANG, SIZE_BOOK_TITLE, SIZE_BOOK_ANNOTATION

from opds_catalog.models import SIZE_CAT_CATNAME, SIZE_CAT_PATH, SIZE_AUTHOR_NAME, SIZE_GENRE, SIZE_GENRE_SECTION, SIZE_GENRE_SUBSECTION, SIZE_SERIES



##########################################################################

# типы каталогов (cat_type)
@@ -74,11 +76,6 @@ def books_del_phisical(): 
    # sql='delete from '+TBL_BGENRES+' where book_id in (select book_id from '+TBL_BOOKS+' where avail<=1)'

    return row_count



def zipisscanned(zipname):

    row_count = Book.objects.filter(path=zipname).update(avail=2)

    return row_count





def arc_skip(arcpath,arcsize):

    """

       Выясняем изменялся ли архив (ZIP или INP-файл)
@@ -137,7 +134,7 @@ def inpx_skip(arcpath,arcsize): 
def findcat(cat_name):

    (head,tail)=os.path.split(cat_name)

    try:

        catalog = Catalog.objects.get(cat_name=tail, path=cat_name)

        catalog = Catalog.objects.get(cat_name=tail[:SIZE_CAT_CATNAME], path=cat_name[:SIZE_CAT_PATH])

    except Catalog.DoesNotExist:

        catalog = None
@@ -151,7 +148,7 @@ def addcattree(cat_name, archive=0, size = 0): 
        return Catalog.objects.get_or_create(parent=None, cat_name=".", path=".", cat_type=0)[0]

    (head,tail)=os.path.split(cat_name)

    parent=addcattree(head)

    new_cat = Catalog.objects.create(parent=parent, cat_name=tail, path=cat_name, cat_type=archive, cat_size=size)

    new_cat = Catalog.objects.create(parent=parent, cat_name=tail[:SIZE_CAT_CATNAME], path=cat_name[:SIZE_CAT_PATH], cat_type=archive, cat_size=size)



    return new_cat
@@ -160,7 +157,7 @@ def findbook(name, path, setavail=0): 
    # а книга была восстановлена в своем старом месте

    # то произойдет восстановление записи об этой книги а не добавится новая

    try:

        book = Book.objects.get(filename=name, path=path)

        book = Book.objects.get(filename=name[:SIZE_BOOK_FILENAME], path=path[:SIZE_BOOK_PATH])

    except Book.DoesNotExist:

        book = None
@@ -170,35 +167,23 @@ def findbook(name, path, setavail=0): 


    return book





def addbook(name, path, cat, exten, title, annotation, docdate, lang, size=0, archive=0):

    book = Book.objects.create(filename=name,path=path,catalog=cat,filesize=size,format=exten.lower()[:8],

                title=title[:256],search_title=title.upper()[:256],annotation=annotation[:10000],docdate=docdate[:32],lang=lang[:16],

                cat_type=archive,avail=2, lang_code=getlangcode(title))

    book = Book.objects.create(filename=name[:SIZE_BOOK_FILENAME],path=path[:SIZE_BOOK_PATH],catalog=cat,filesize=size,format=exten.lower()[:SIZE_BOOK_FORMAT],

                title=title[:SIZE_BOOK_TITLE],search_title=title.upper()[:SIZE_BOOK_TITLE],annotation=annotation[:SIZE_BOOK_ANNOTATION],

                docdate=docdate[:SIZE_BOOK_DOCDATE],lang=lang[:SIZE_BOOK_LANG],cat_type=archive,avail=2, lang_code=getlangcode(title))

    return book



#def findauthor(first_name,last_name):

#    try:

#        author = Author.objects.filter(last_name=last_name, first_name=first_name)[:1]

#    except Author.DoesNotExist:

#        author = None

#

#    return author



#def addauthor(first_name, last_name):

#    author, created = Author.objects.get_or_create(last_name=last_name, first_name=first_name, lang_code=getlangcode(last_name))

#    return author



def findauthor(full_name):

    try:

        author = Author.objects.filter(full_name=full_name[:128])[:1]

        author = Author.objects.filter(full_name=full_name[:SIZE_AUTHOR_NAME])[:1]

    except Author.DoesNotExist:

        author = None



    return author



def addauthor(full_name):

    author, created = Author.objects.get_or_create(full_name=full_name[:128], defaults={'search_full_name':full_name.upper()[:128], 'lang_code':getlangcode(full_name)})

    author, created = Author.objects.get_or_create(full_name=full_name[:SIZE_AUTHOR_NAME], defaults={'search_full_name':full_name.upper()[:SIZE_AUTHOR_NAME], 

                                                                                                     'lang_code':getlangcode(full_name)})

    return author



def addbauthor(book, author):
@@ -206,7 +191,7 @@ def addbauthor(book, author): 
    ba.save()



def addgenre(genre):

    genre, created = Genre.objects.get_or_create(genre=genre[:32], defaults={'section':unknown_genre, 'subsection':genre[:100]})

    genre, created = Genre.objects.get_or_create(genre=genre[:SIZE_GENRE], defaults={'section':unknown_genre, 'subsection':genre[:SIZE_GENRE_SUBSECTION]})

    return genre



def addbgenre(book, genre):
@@ -214,7 +199,7 @@ def addbgenre(book, genre): 
    bg.save()



def addseries(ser):

    series, created = Series.objects.get_or_create(ser=ser[:80], defaults={'search_ser':ser.upper()[:80], 'lang_code':getlangcode(ser)})

    series, created = Series.objects.get_or_create(ser=ser[:SIZE_SERIES], defaults={'search_ser':ser.upper()[:80], 'lang_code':getlangcode(ser)})

    return series



def addbseries(book, ser, ser_no):

sopds_web_backend/views.py

+6 −6
Original line number Diff line number Diff line
@@ -70,13 +70,13 @@ def SearchBooksView(request): 
        

        if searchtype == 'm':

            #books = Book.objects.extra(where=["upper(title) like %s"], params=["%%%s%%"%searchterms.upper()]).order_by('title','-docdate')

            books = Book.objects.filter(search_title__contains=searchterms.upper()).order_by('title','-docdate')

            books = Book.objects.filter(search_title__contains=searchterms.upper()).order_by('search_title','-docdate')

            args['breadcrumbs'] = [_('Books'),_('Search by title'),searchterms]

            args['searchobject'] = 'title'

            

        if searchtype == 'b':

            #books = Book.objects.extra(where=["upper(title) like %s"], params=["%s%%"%searchterms.upper()]).order_by('title','-docdate')

            books = Book.objects.filter(search_title__startswith=searchterms.upper()).order_by('title','-docdate')

            books = Book.objects.filter(search_title__startswith=searchterms.upper()).order_by('search_title','-docdate')

            args['breadcrumbs'] = [_('Books'),_('Search by title'),searchterms]   

            args['searchobject'] = 'title'
@@ -89,7 +89,7 @@ def SearchBooksView(request): 
            except:

                author_id = 0

                aname = ""                  

            books = Book.objects.filter(authors=author_id).order_by('title','-docdate')  

            books = Book.objects.filter(authors=author_id).order_by('search_title','-docdate')  

            args['breadcrumbs'] = [_('Books'),_('Search by author'),aname]   

            args['searchobject'] = 'author'
@@ -101,7 +101,7 @@ def SearchBooksView(request): 
            except:

                ser_id = 0

                ser = ""

            books = Book.objects.filter(series=ser_id).order_by('title','-docdate')    

            books = Book.objects.filter(series=ser_id).order_by('search_title','-docdate')    

            args['breadcrumbs'] = [_('Books'),_('Search by series'),ser]

            args['searchobject'] = 'series'
@@ -116,7 +116,7 @@ def SearchBooksView(request): 
                genre_id = 0

                args['breadcrumbs'] = [_('Books'),_('Search by genre')]

                

            books = Book.objects.filter(genres=genre_id).order_by('title','-docdate') 

            books = Book.objects.filter(genres=genre_id).order_by('search_title','-docdate') 

            args['searchobject'] = 'genre'

                                   

        # Поиск книг на книжной полке
@@ -341,7 +341,7 @@ def CatalogsView(request): 
    except Catalog.DoesNotExist:

        cat = None

    

    catalogs_list = Catalog.objects.filter(parent=cat).order_by("cat_type","cat_name")

    catalogs_list = Catalog.objects.filter(parent=cat).order_by("cat_name")

    # prefetch_related on sqlite on items >999 therow error "too many SQL variables"

    #books_list = Book.objects.filter(catalog=cat).prefetch_related('authors','genres','series').order_by("title")

    books_list = Book.objects.filter(catalog=cat).order_by("search_title")