Separate out the hashtag swarm

main
Bob Mottram 2020-11-19 14:02:16 +00:00
parent 7c52631725
commit 5357bc8223
2 changed files with 294 additions and 283 deletions

View File

@ -0,0 +1,86 @@
__filename__ = "webapp_hashtagswarm.py"
__author__ = "Bob Mottram"
__license__ = "AGPL3+"
__version__ = "1.1.0"
__maintainer__ = "Bob Mottram"
__email__ = "bob@freedombone.net"
__status__ = "Production"
import os
from blocking import isBlockedHashtag
from datetime import datetime
def htmlHashTagSwarm(baseDir: str, actor: str) -> str:
"""Returns a tag swarm of today's hashtags
"""
currTime = datetime.utcnow()
daysSinceEpoch = (currTime - datetime(1970, 1, 1)).days
daysSinceEpochStr = str(daysSinceEpoch) + ' '
tagSwarm = []
for subdir, dirs, files in os.walk(baseDir + '/tags'):
for f in files:
tagsFilename = os.path.join(baseDir + '/tags', f)
if not os.path.isfile(tagsFilename):
continue
# get last modified datetime
modTimesinceEpoc = os.path.getmtime(tagsFilename)
lastModifiedDate = datetime.fromtimestamp(modTimesinceEpoc)
fileDaysSinceEpoch = (lastModifiedDate - datetime(1970, 1, 1)).days
# check if the file was last modified today
if fileDaysSinceEpoch != daysSinceEpoch:
continue
hashTagName = f.split('.')[0]
if isBlockedHashtag(baseDir, hashTagName):
continue
if daysSinceEpochStr not in open(tagsFilename).read():
continue
with open(tagsFilename, 'r') as tagsFile:
line = tagsFile.readline()
lineCtr = 1
tagCtr = 0
maxLineCtr = 1
while line:
if ' ' not in line:
line = tagsFile.readline()
lineCtr += 1
# don't read too many lines
if lineCtr >= maxLineCtr:
break
continue
postDaysSinceEpochStr = line.split(' ')[0]
if not postDaysSinceEpochStr.isdigit():
line = tagsFile.readline()
lineCtr += 1
# don't read too many lines
if lineCtr >= maxLineCtr:
break
continue
postDaysSinceEpoch = int(postDaysSinceEpochStr)
if postDaysSinceEpoch < daysSinceEpoch:
break
if postDaysSinceEpoch == daysSinceEpoch:
if tagCtr == 0:
tagSwarm.append(hashTagName)
tagCtr += 1
line = tagsFile.readline()
lineCtr += 1
# don't read too many lines
if lineCtr >= maxLineCtr:
break
if not tagSwarm:
return ''
tagSwarm.sort()
tagSwarmStr = ''
ctr = 0
for tagName in tagSwarm:
tagSwarmStr += \
'<a href="' + actor + '/tags/' + tagName + \
'" class="hashtagswarm">' + tagName + '</a>\n'
ctr += 1
tagSwarmHtml = tagSwarmStr.strip() + '\n'
return tagSwarmHtml

View File

@ -28,7 +28,7 @@ from webapp_utils import htmlFooter
from webapp_utils import getSearchBannerFile from webapp_utils import getSearchBannerFile
from webapp_utils import htmlPostSeparator from webapp_utils import htmlPostSeparator
from webapp_post import individualPostAsHtml from webapp_post import individualPostAsHtml
from blocking import isBlockedHashtag from webapp_hashtagswarm import htmlHashTagSwarm
def htmlSearchEmoji(cssCache: {}, translate: {}, def htmlSearchEmoji(cssCache: {}, translate: {},
@ -380,79 +380,221 @@ def htmlSearch(cssCache: {}, translate: {},
return followStr return followStr
def htmlHashTagSwarm(baseDir: str, actor: str) -> str: def htmlSkillsSearch(cssCache: {}, translate: {}, baseDir: str,
"""Returns a tag swarm of today's hashtags httpPrefix: str,
skillsearch: str, instanceOnly: bool,
postsPerPage: int) -> str:
"""Show a page containing search results for a skill
""" """
currTime = datetime.utcnow() if skillsearch.startswith('*'):
daysSinceEpoch = (currTime - datetime(1970, 1, 1)).days skillsearch = skillsearch[1:].strip()
daysSinceEpochStr = str(daysSinceEpoch) + ' '
tagSwarm = []
for subdir, dirs, files in os.walk(baseDir + '/tags'): skillsearch = skillsearch.lower().strip('\n').strip('\r')
results = []
# search instance accounts
for subdir, dirs, files in os.walk(baseDir + '/accounts/'):
for f in files: for f in files:
tagsFilename = os.path.join(baseDir + '/tags', f) if not f.endswith('.json'):
if not os.path.isfile(tagsFilename):
continue continue
# get last modified datetime if '@' not in f:
modTimesinceEpoc = os.path.getmtime(tagsFilename)
lastModifiedDate = datetime.fromtimestamp(modTimesinceEpoc)
fileDaysSinceEpoch = (lastModifiedDate - datetime(1970, 1, 1)).days
# check if the file was last modified today
if fileDaysSinceEpoch != daysSinceEpoch:
continue continue
if f.startswith('inbox@'):
continue
actorFilename = os.path.join(subdir, f)
actorJson = loadJson(actorFilename)
if actorJson:
if actorJson.get('id') and \
actorJson.get('skills') and \
actorJson.get('name') and \
actorJson.get('icon'):
actor = actorJson['id']
for skillName, skillLevel in actorJson['skills'].items():
skillName = skillName.lower()
if not (skillName in skillsearch or
skillsearch in skillName):
continue
skillLevelStr = str(skillLevel)
if skillLevel < 100:
skillLevelStr = '0' + skillLevelStr
if skillLevel < 10:
skillLevelStr = '0' + skillLevelStr
indexStr = \
skillLevelStr + ';' + actor + ';' + \
actorJson['name'] + \
';' + actorJson['icon']['url']
if indexStr not in results:
results.append(indexStr)
if not instanceOnly:
# search actor cache
for subdir, dirs, files in os.walk(baseDir + '/cache/actors/'):
for f in files:
if not f.endswith('.json'):
continue
if '@' not in f:
continue
if f.startswith('inbox@'):
continue
actorFilename = os.path.join(subdir, f)
cachedActorJson = loadJson(actorFilename)
if cachedActorJson:
if cachedActorJson.get('actor'):
actorJson = cachedActorJson['actor']
if actorJson.get('id') and \
actorJson.get('skills') and \
actorJson.get('name') and \
actorJson.get('icon'):
actor = actorJson['id']
for skillName, skillLevel in \
actorJson['skills'].items():
skillName = skillName.lower()
if not (skillName in skillsearch or
skillsearch in skillName):
continue
skillLevelStr = str(skillLevel)
if skillLevel < 100:
skillLevelStr = '0' + skillLevelStr
if skillLevel < 10:
skillLevelStr = '0' + skillLevelStr
indexStr = \
skillLevelStr + ';' + actor + ';' + \
actorJson['name'] + \
';' + actorJson['icon']['url']
if indexStr not in results:
results.append(indexStr)
hashTagName = f.split('.')[0] results.sort(reverse=True)
if isBlockedHashtag(baseDir, hashTagName):
continue
if daysSinceEpochStr not in open(tagsFilename).read():
continue
with open(tagsFilename, 'r') as tagsFile:
line = tagsFile.readline()
lineCtr = 1
tagCtr = 0
maxLineCtr = 1
while line:
if ' ' not in line:
line = tagsFile.readline()
lineCtr += 1
# don't read too many lines
if lineCtr >= maxLineCtr:
break
continue
postDaysSinceEpochStr = line.split(' ')[0]
if not postDaysSinceEpochStr.isdigit():
line = tagsFile.readline()
lineCtr += 1
# don't read too many lines
if lineCtr >= maxLineCtr:
break
continue
postDaysSinceEpoch = int(postDaysSinceEpochStr)
if postDaysSinceEpoch < daysSinceEpoch:
break
if postDaysSinceEpoch == daysSinceEpoch:
if tagCtr == 0:
tagSwarm.append(hashTagName)
tagCtr += 1
line = tagsFile.readline() cssFilename = baseDir + '/epicyon-profile.css'
lineCtr += 1 if os.path.isfile(baseDir + '/epicyon.css'):
# don't read too many lines cssFilename = baseDir + '/epicyon.css'
if lineCtr >= maxLineCtr:
break
if not tagSwarm: skillSearchForm = htmlHeaderWithExternalStyle(cssFilename)
return '' skillSearchForm += \
tagSwarm.sort() '<center><h1>' + translate['Skills search'] + ': ' + \
tagSwarmStr = '' skillsearch + '</h1></center>'
if len(results) == 0:
skillSearchForm += \
'<center><h5>' + translate['No results'] + \
'</h5></center>'
else:
skillSearchForm += '<center>'
ctr = 0 ctr = 0
for tagName in tagSwarm: for skillMatch in results:
tagSwarmStr += \ skillMatchFields = skillMatch.split(';')
'<a href="' + actor + '/tags/' + tagName + \ if len(skillMatchFields) != 4:
'" class="hashtagswarm">' + tagName + '</a>\n' continue
actor = skillMatchFields[1]
actorName = skillMatchFields[2]
avatarUrl = skillMatchFields[3]
skillSearchForm += \
'<div class="search-result""><a href="' + \
actor + '/skills">'
skillSearchForm += \
'<img loading="lazy" src="' + avatarUrl + \
'"/><span class="search-result-text">' + actorName + \
'</span></a></div>'
ctr += 1 ctr += 1
tagSwarmHtml = tagSwarmStr.strip() + '\n' if ctr >= postsPerPage:
return tagSwarmHtml break
skillSearchForm += '</center>'
skillSearchForm += htmlFooter()
return skillSearchForm
def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str,
httpPrefix: str,
nickname: str, domain: str,
historysearch: str,
postsPerPage: int, pageNumber: int,
projectVersion: str,
recentPostsCache: {},
maxRecentPosts: int,
session,
wfRequest,
personCache: {},
port: int,
YTReplacementDomain: str,
showPublishedDateOnly: bool) -> str:
"""Show a page containing search results for your post history
"""
if historysearch.startswith('!'):
historysearch = historysearch[1:].strip()
historysearch = historysearch.lower().strip('\n').strip('\r')
boxFilenames = \
searchBoxPosts(baseDir, nickname, domain,
historysearch, postsPerPage)
cssFilename = baseDir + '/epicyon-profile.css'
if os.path.isfile(baseDir + '/epicyon.css'):
cssFilename = baseDir + '/epicyon.css'
historySearchForm = \
htmlHeaderWithExternalStyle(cssFilename)
# add the page title
historySearchForm += \
'<center><h1>' + translate['Your Posts'] + '</h1></center>'
if len(boxFilenames) == 0:
historySearchForm += \
'<center><h5>' + translate['No results'] + \
'</h5></center>'
return historySearchForm
iconsPath = getIconsWebPath(baseDir)
separatorStr = htmlPostSeparator(baseDir, None)
# ensure that the page number is in bounds
if not pageNumber:
pageNumber = 1
elif pageNumber < 1:
pageNumber = 1
# get the start end end within the index file
startIndex = int((pageNumber - 1) * postsPerPage)
endIndex = startIndex + postsPerPage
noOfBoxFilenames = len(boxFilenames)
if endIndex >= noOfBoxFilenames and noOfBoxFilenames > 0:
endIndex = noOfBoxFilenames - 1
index = startIndex
while index <= endIndex:
postFilename = boxFilenames[index]
if not postFilename:
index += 1
continue
postJsonObject = loadJson(postFilename)
if not postJsonObject:
index += 1
continue
showIndividualPostIcons = True
allowDeletion = False
postStr = \
individualPostAsHtml(True, recentPostsCache,
maxRecentPosts,
iconsPath, translate, None,
baseDir, session, wfRequest,
personCache,
nickname, domain, port,
postJsonObject,
None, True, allowDeletion,
httpPrefix, projectVersion,
'search',
YTReplacementDomain,
showPublishedDateOnly,
showIndividualPostIcons,
showIndividualPostIcons,
False, False, False)
if postStr:
historySearchForm += separatorStr + postStr
index += 1
historySearchForm += htmlFooter()
return historySearchForm
def htmlHashtagSearch(cssCache: {}, def htmlHashtagSearch(cssCache: {},
@ -711,220 +853,3 @@ def rssHashtagSearch(nickname: str, domain: str, port: int,
break break
return hashtagFeed + rss2TagFooter() return hashtagFeed + rss2TagFooter()
def htmlSkillsSearch(cssCache: {}, translate: {}, baseDir: str,
httpPrefix: str,
skillsearch: str, instanceOnly: bool,
postsPerPage: int) -> str:
"""Show a page containing search results for a skill
"""
if skillsearch.startswith('*'):
skillsearch = skillsearch[1:].strip()
skillsearch = skillsearch.lower().strip('\n').strip('\r')
results = []
# search instance accounts
for subdir, dirs, files in os.walk(baseDir + '/accounts/'):
for f in files:
if not f.endswith('.json'):
continue
if '@' not in f:
continue
if f.startswith('inbox@'):
continue
actorFilename = os.path.join(subdir, f)
actorJson = loadJson(actorFilename)
if actorJson:
if actorJson.get('id') and \
actorJson.get('skills') and \
actorJson.get('name') and \
actorJson.get('icon'):
actor = actorJson['id']
for skillName, skillLevel in actorJson['skills'].items():
skillName = skillName.lower()
if not (skillName in skillsearch or
skillsearch in skillName):
continue
skillLevelStr = str(skillLevel)
if skillLevel < 100:
skillLevelStr = '0' + skillLevelStr
if skillLevel < 10:
skillLevelStr = '0' + skillLevelStr
indexStr = \
skillLevelStr + ';' + actor + ';' + \
actorJson['name'] + \
';' + actorJson['icon']['url']
if indexStr not in results:
results.append(indexStr)
if not instanceOnly:
# search actor cache
for subdir, dirs, files in os.walk(baseDir + '/cache/actors/'):
for f in files:
if not f.endswith('.json'):
continue
if '@' not in f:
continue
if f.startswith('inbox@'):
continue
actorFilename = os.path.join(subdir, f)
cachedActorJson = loadJson(actorFilename)
if cachedActorJson:
if cachedActorJson.get('actor'):
actorJson = cachedActorJson['actor']
if actorJson.get('id') and \
actorJson.get('skills') and \
actorJson.get('name') and \
actorJson.get('icon'):
actor = actorJson['id']
for skillName, skillLevel in \
actorJson['skills'].items():
skillName = skillName.lower()
if not (skillName in skillsearch or
skillsearch in skillName):
continue
skillLevelStr = str(skillLevel)
if skillLevel < 100:
skillLevelStr = '0' + skillLevelStr
if skillLevel < 10:
skillLevelStr = '0' + skillLevelStr
indexStr = \
skillLevelStr + ';' + actor + ';' + \
actorJson['name'] + \
';' + actorJson['icon']['url']
if indexStr not in results:
results.append(indexStr)
results.sort(reverse=True)
cssFilename = baseDir + '/epicyon-profile.css'
if os.path.isfile(baseDir + '/epicyon.css'):
cssFilename = baseDir + '/epicyon.css'
skillSearchForm = htmlHeaderWithExternalStyle(cssFilename)
skillSearchForm += \
'<center><h1>' + translate['Skills search'] + ': ' + \
skillsearch + '</h1></center>'
if len(results) == 0:
skillSearchForm += \
'<center><h5>' + translate['No results'] + \
'</h5></center>'
else:
skillSearchForm += '<center>'
ctr = 0
for skillMatch in results:
skillMatchFields = skillMatch.split(';')
if len(skillMatchFields) != 4:
continue
actor = skillMatchFields[1]
actorName = skillMatchFields[2]
avatarUrl = skillMatchFields[3]
skillSearchForm += \
'<div class="search-result""><a href="' + \
actor + '/skills">'
skillSearchForm += \
'<img loading="lazy" src="' + avatarUrl + \
'"/><span class="search-result-text">' + actorName + \
'</span></a></div>'
ctr += 1
if ctr >= postsPerPage:
break
skillSearchForm += '</center>'
skillSearchForm += htmlFooter()
return skillSearchForm
def htmlHistorySearch(cssCache: {}, translate: {}, baseDir: str,
httpPrefix: str,
nickname: str, domain: str,
historysearch: str,
postsPerPage: int, pageNumber: int,
projectVersion: str,
recentPostsCache: {},
maxRecentPosts: int,
session,
wfRequest,
personCache: {},
port: int,
YTReplacementDomain: str,
showPublishedDateOnly: bool) -> str:
"""Show a page containing search results for your post history
"""
if historysearch.startswith('!'):
historysearch = historysearch[1:].strip()
historysearch = historysearch.lower().strip('\n').strip('\r')
boxFilenames = \
searchBoxPosts(baseDir, nickname, domain,
historysearch, postsPerPage)
cssFilename = baseDir + '/epicyon-profile.css'
if os.path.isfile(baseDir + '/epicyon.css'):
cssFilename = baseDir + '/epicyon.css'
historySearchForm = \
htmlHeaderWithExternalStyle(cssFilename)
# add the page title
historySearchForm += \
'<center><h1>' + translate['Your Posts'] + '</h1></center>'
if len(boxFilenames) == 0:
historySearchForm += \
'<center><h5>' + translate['No results'] + \
'</h5></center>'
return historySearchForm
iconsPath = getIconsWebPath(baseDir)
separatorStr = htmlPostSeparator(baseDir, None)
# ensure that the page number is in bounds
if not pageNumber:
pageNumber = 1
elif pageNumber < 1:
pageNumber = 1
# get the start end end within the index file
startIndex = int((pageNumber - 1) * postsPerPage)
endIndex = startIndex + postsPerPage
noOfBoxFilenames = len(boxFilenames)
if endIndex >= noOfBoxFilenames and noOfBoxFilenames > 0:
endIndex = noOfBoxFilenames - 1
index = startIndex
while index <= endIndex:
postFilename = boxFilenames[index]
if not postFilename:
index += 1
continue
postJsonObject = loadJson(postFilename)
if not postJsonObject:
index += 1
continue
showIndividualPostIcons = True
allowDeletion = False
postStr = \
individualPostAsHtml(True, recentPostsCache,
maxRecentPosts,
iconsPath, translate, None,
baseDir, session, wfRequest,
personCache,
nickname, domain, port,
postJsonObject,
None, True, allowDeletion,
httpPrefix, projectVersion,
'search',
YTReplacementDomain,
showPublishedDateOnly,
showIndividualPostIcons,
showIndividualPostIcons,
False, False, False)
if postStr:
historySearchForm += separatorStr + postStr
index += 1
historySearchForm += htmlFooter()
return historySearchForm