snuffster/szurubooru
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server/tools: add parallelism to content import

Browse source
This commit is contained in:
rr- 2016-05-09 09:42:31 +02:00
parent 356d5050fd
commit 4558557656
1 changed files with 35 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
server/migrate-v1
View file

@ -5,6 +5,7 @@ import datetime
import argparse import argparse
import json import json
import zlib import zlib
import concurrent.futures
import logging import logging
import coloredlogs import coloredlogs
import sqlalchemy import sqlalchemy
@ -190,31 +191,41 @@ def import_posts(v1_session, v2_session):
v2_session.commit() v2_session.commit()
return unused_post_ids return unused_post_ids
def _import_post_content_for_post(
unused_post_ids, v1_data_dir, v1_session, v2_session, row, post):
logger.info('Importing post %d content...', row['id'])
if row['id'] in unused_post_ids:
logger.warn('Ignoring unimported post %d', row['id'])
return
assert post
source_content_path = os.path.join(
v1_data_dir,
'public_html',
'data',
'posts',
row['name'])
source_thumb_path = os.path.join(
v1_data_dir,
'public_html',
'data',
'posts',
row['name'] + '-custom-thumb')
post_content = read_file(source_content_path)
files.save(posts.get_post_content_path(post), post_content)
if os.path.exists(source_thumb_path):
thumb_content = read_file(source_thumb_path)
files.save(posts.get_post_thumbnail_backup_path(post), thumb_content)
posts.generate_post_thumbnail(post)
def import_post_content(unused_post_ids, v1_data_dir, v1_session, v2_session): def import_post_content(unused_post_ids, v1_data_dir, v1_session, v2_session):
for row in exec(v1_session, 'SELECT * FROM posts'): rows = list(exec(v1_session, 'SELECT * FROM posts'))
logger.info('Importing post %d content...', row['id']) posts = {post.post_id: post for post in v2_session.query(db.Post).all()}
if row['id'] in unused_post_ids: with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
logger.warn('Ignoring unimported post %d', row['id']) for row in rows:
continue post = posts.get(row['id'])
post = posts.get_post_by_id(row['id']) executor.submit(
source_content_path = os.path.join( _import_post_content_for_post,
v1_data_dir, unused_post_ids, v1_data_dir, v1_session, v2_session, row, post)
'public_html',
'data',
'posts',
row['name'])
source_thumb_path = os.path.join(
v1_data_dir,
'public_html',
'data',
'posts',
row['name'] + '-custom-thumb')
post_content = read_file(source_content_path)
files.save(posts.get_post_content_path(post), post_content)
if os.path.exists(source_thumb_path):
thumb_content = read_file(source_thumb_path)
files.save(posts.get_post_thumbnail_backup_path(post), thumb_content)
posts.generate_post_thumbnail(post)
def import_post_tags(unused_post_ids, v1_session, v2_session): def import_post_tags(unused_post_ids, v1_session, v2_session):
logger.info('Importing post tags...') logger.info('Importing post tags...')