diff --git a/posts.py b/posts.py index bebd55cc4..8ad328394 100644 --- a/posts.py +++ b/posts.py @@ -3980,112 +3980,115 @@ def _create_box_items(base_dir: str, '/' + index_box_name + '.index' total_posts_count = 0 posts_added_to_timeline = 0 - if os.path.isfile(index_filename): - if first_post_id: - first_post_id = first_post_id.replace('--', '#') - first_post_id = first_post_id.replace('/', '#') - with open(index_filename, 'r', encoding='utf-8') as index_file: - posts_added_to_timeline = 0 - while posts_added_to_timeline < items_per_page: - post_filename = index_file.readline() + if not os.path.isfile(index_filename): + return total_posts_count, posts_added_to_timeline - if not post_filename: - break + if first_post_id: + first_post_id = first_post_id.replace('--', '#') + first_post_id = first_post_id.replace('/', '#') - if first_post_id and total_posts_count == 0: - if first_post_id not in post_filename: - continue - total_posts_count = \ - int((page_number - 1) * items_per_page) + with open(index_filename, 'r', encoding='utf-8') as index_file: + posts_added_to_timeline = 0 + while posts_added_to_timeline < items_per_page: + post_filename = index_file.readline() - # Has this post passed through the newswire voting stage? - if not _passed_newswire_voting(newswire_votes_threshold, - base_dir, domain, - post_filename, - positive_voting, - voting_time_mins): + if not post_filename: + break + + if first_post_id and total_posts_count == 0: + if first_post_id not in post_filename: + continue + total_posts_count = \ + int((page_number - 1) * items_per_page) + + # Has this post passed through the newswire voting stage? + if not _passed_newswire_voting(newswire_votes_threshold, + base_dir, domain, + post_filename, + positive_voting, + voting_time_mins): + continue + + # Skip through any posts previous to the current page + if not first_post_id: + if total_posts_count < \ + int((page_number - 1) * items_per_page): + total_posts_count += 1 continue - # Skip through any posts previous to the current page - if not first_post_id: - if total_posts_count < \ - int((page_number - 1) * items_per_page): - total_posts_count += 1 - continue + # if this is a full path then remove the directories + if '/' in post_filename: + post_filename = post_filename.split('/')[-1] - # if this is a full path then remove the directories - if '/' in post_filename: - post_filename = post_filename.split('/')[-1] + # filename of the post without any extension or path + # This should also correspond to any index entry in + # the posts cache + post_url = remove_eol(post_filename) + post_url = post_url.replace('.json', '').strip() - # filename of the post without any extension or path - # This should also correspond to any index entry in - # the posts cache - post_url = remove_eol(post_filename) - post_url = post_url.replace('.json', '').strip() + if post_url in post_urls_in_box: + continue - if post_url in post_urls_in_box: + # is the post cached in memory? + if recent_posts_cache.get('index'): + if post_url in recent_posts_cache['index']: + if recent_posts_cache['json'].get(post_url): + url = recent_posts_cache['json'][post_url] + if _add_post_string_to_timeline(url, + boxname, + posts_in_box, + box_actor): + total_posts_count += 1 + posts_added_to_timeline += 1 + post_urls_in_box.append(post_url) + continue + print('Post not added to timeline') + + # read the post from file + full_post_filename = \ + locate_post(base_dir, nickname, + original_domain, post_url, False) + if full_post_filename: + # has the post been rejected? + if os.path.isfile(full_post_filename + '.reject'): continue - # is the post cached in memory? - if recent_posts_cache.get('index'): - if post_url in recent_posts_cache['index']: - if recent_posts_cache['json'].get(post_url): - url = recent_posts_cache['json'][post_url] - if _add_post_string_to_timeline(url, - boxname, - posts_in_box, - box_actor): - total_posts_count += 1 - posts_added_to_timeline += 1 - post_urls_in_box.append(post_url) - continue - print('Post not added to timeline') - - # read the post from file - full_post_filename = \ - locate_post(base_dir, nickname, - original_domain, post_url, False) - if full_post_filename: - # has the post been rejected? - if os.path.isfile(full_post_filename + '.reject'): - continue - - if _add_post_to_timeline(full_post_filename, boxname, - posts_in_box, box_actor): - posts_added_to_timeline += 1 - total_posts_count += 1 - post_urls_in_box.append(post_url) - else: - print('WARN: Unable to add post ' + post_url + - ' nickname ' + nickname + - ' timeline ' + boxname) + if _add_post_to_timeline(full_post_filename, boxname, + posts_in_box, box_actor): + posts_added_to_timeline += 1 + total_posts_count += 1 + post_urls_in_box.append(post_url) else: - if timeline_nickname != nickname: - # if this is the features timeline - full_post_filename = \ - locate_post(base_dir, timeline_nickname, - original_domain, post_url, False) - if full_post_filename: - if _add_post_to_timeline(full_post_filename, - boxname, - posts_in_box, box_actor): - posts_added_to_timeline += 1 - total_posts_count += 1 - post_urls_in_box.append(post_url) - else: - print('WARN: Unable to add features post ' + - post_url + ' nickname ' + nickname + - ' timeline ' + boxname) + print('WARN: Unable to add post ' + post_url + + ' nickname ' + nickname + + ' timeline ' + boxname) + else: + if timeline_nickname != nickname: + # if this is the features timeline + full_post_filename = \ + locate_post(base_dir, timeline_nickname, + original_domain, post_url, False) + if full_post_filename: + if _add_post_to_timeline(full_post_filename, + boxname, + posts_in_box, box_actor): + posts_added_to_timeline += 1 + total_posts_count += 1 + post_urls_in_box.append(post_url) else: - print('WARN: features timeline. ' + - 'Unable to locate post ' + post_url) + print('WARN: Unable to add features post ' + + post_url + ' nickname ' + nickname + + ' timeline ' + boxname) else: - if timeline_nickname == 'news': - print('WARN: Unable to locate news post ' + - post_url + ' nickname ' + nickname) - else: - print('WARN: Unable to locate post ' + post_url + - ' nickname ' + nickname) + print('WARN: features timeline. ' + + 'Unable to locate post ' + post_url) + else: + if timeline_nickname == 'news': + print('WARN: Unable to locate news post ' + + post_url + ' nickname ' + nickname) + else: + print('WARN: Unable to locate post ' + post_url + + ' nickname ' + nickname) return total_posts_count, posts_added_to_timeline