mj-saunders
/
epicyon
forked from indymedia/epicyon
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Store recent posts in memory for fast access

main
Bob Mottram 2019-11-24 11:28:58 +00:00
parent 1fb7811d08
commit bd68e84776
5 changed files with 152 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
daemon.py
View File

@ -2420,7 +2420,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInFeed, 'inbox', \
authorized,self.server.ocapAlways)
msg=htmlInbox(self.server.translate, \
msg=htmlInbox(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2494,7 +2495,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInFeed, 'dm', \
authorized,self.server.ocapAlways)
msg=htmlInboxDMs(self.server.translate, \
msg=htmlInboxDMs(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2569,7 +2571,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInFeed, 'tlreplies', \
True,self.server.ocapAlways)
msg=htmlInboxReplies(self.server.translate, \
msg=htmlInboxReplies(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2644,7 +2647,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInMediaFeed, 'tlmedia', \
True,self.server.ocapAlways)
msg=htmlInboxMedia(self.server.translate, \
msg=htmlInboxMedia(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInMediaFeed, \
self.server.session, \
self.server.baseDir, \
@ -2697,7 +2701,8 @@ class PubServer(BaseHTTPRequestHandler):
pageNumber=int(pageNumber)
else:
pageNumber=1
msg=htmlShares(self.server.translate, \
msg=htmlShares(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2756,7 +2761,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInFeed, 'tlbookmarks', \
authorized,self.server.ocapAlways)
msg=htmlBookmarks(self.server.translate, \
msg=htmlBookmarks(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2826,7 +2832,8 @@ class PubServer(BaseHTTPRequestHandler):
maxPostsInFeed, 'outbox', \
authorized, \
self.server.ocapAlways)
msg=htmlOutbox(self.server.translate, \
msg=htmlOutbox(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -2890,7 +2897,8 @@ class PubServer(BaseHTTPRequestHandler):
self.server.httpPrefix, \
maxPostsInFeed, 'moderation', \
True,self.server.ocapAlways)
msg=htmlModeration(self.server.translate, \
msg=htmlModeration(self.server.recentPostsCache, \
self.server.translate, \
pageNumber,maxPostsInFeed, \
self.server.session, \
self.server.baseDir, \
@ -5092,7 +5100,8 @@ def loadTokens(baseDir: str,tokensDict: {},tokensLookup: {}) -> None:
tokensDict[nickname]=token
tokensLookup[token]=nickname
def runDaemon(enableSharedInbox: bool,registration: bool, \
def runDaemon(maxRecentPosts: int, \
enableSharedInbox: bool,registration: bool, \
language: str,projectVersion: str, \
instanceId: str,clientToServer: bool, \
baseDir: str,domain: str, \
@ -5261,10 +5270,14 @@ def runDaemon(enableSharedInbox: bool,registration: bool, \
else:
httpd.thrSharesExpire.start()
httpd.recentPostsCache={}
httpd.maxRecentPosts=maxRecentPosts
print('Creating inbox queue')
httpd.thrInboxQueue= \
threadWithTrace(target=runInboxQueue, \
args=(projectVersion, \
args=(httpd.recentPostsCache,httpd.maxRecentPosts, \
projectVersion, \
baseDir,httpPrefix,httpd.sendThreads, \
httpd.postLog,httpd.cachedWebfingers, \
httpd.personCache,httpd.inboxQueue, \

4
epicyon.py
View File

@ -104,6 +104,8 @@ parser.add_argument('-d','--domain', dest='domain', type=str,default=None, \
help='Domain name of the server')
parser.add_argument('-p','--port', dest='port', type=int,default=None, \
help='Port number to run on')
parser.add_argument('--postcache', dest='maxRecentPosts', type=int,default=100, \
help='The maximum number of recent posts to store in RAM')
parser.add_argument('--proxy', dest='proxyPort', type=int,default=None, \
help='Proxy port number to run on')
parser.add_argument('--path', dest='baseDir', \
@ -1484,7 +1486,7 @@ if not registration:
if setTheme(baseDir,themeName):
print('Theme set to '+themeName)
runDaemon(not args.nosharedinbox, \
runDaemon(args.maxRecentPosts,not args.nosharedinbox, \
registration,args.language,__version__, \
instanceId,args.client,baseDir, \
domain,port,proxyPort,httpPrefix, \

58
inbox.py
View File

@ -55,7 +55,31 @@ from posts import sendSignedJson
from webinterface import individualPostAsHtml
from webinterface import getIconsDir
def inboxStorePostToHtmlCache(translate: {}, \
def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
postJsonObject: {},htmlStr: str) -> None:
"""Store recent posts in memory so that they can be quickly recalled
"""
if not postJsonObject.get('id'):
return
postId=postJsonObject['id'].replace('/activity','').replace('/','#')
if recentPostsCache.get('index'):
recentPostsCache['index'].append(postId)
recentPostsCache['json'][postId]=postJsonObject.copy()
recentPostsCache['html'][postId]=htmlStr
while len(recentPostsCache['html'].items())>maxRecentPosts:
recentPostsCache['index'].pop(0)
del recentPostsCache['json'][postId]
del recentPostsCache['html'][postId]
else:
recentPostsCache['index']=[postId]
recentPostsCache['json']={}
recentPostsCache['html']={}
recentPostsCache['json'][postId]=postJsonObject.copy()
recentPostsCache['html'][postId]=htmlStr
def inboxStorePostToHtmlCache(recentPostsCache: {},maxRecentPosts: int, \
translate: {}, \
baseDir: str,httpPrefix: str, \
session,cachedWebfingers: {},personCache: {}, \
nickname: str,domain: str,port: int, \
@ -68,13 +92,16 @@ def inboxStorePostToHtmlCache(translate: {}, \
showAvatarOptions=True
avatarUrl=None
boxName='inbox'
individualPostAsHtml(getIconsDir(baseDir),translate,pageNumber, \
baseDir,session,cachedWebfingers,personCache, \
nickname,domain,port,postJsonObject, \
avatarUrl,True,allowDeletion, \
httpPrefix,__version__,boxName, \
not isDM(postJsonObject), \
True,True,False,True)
htmlStr= \
individualPostAsHtml(getIconsDir(baseDir),translate,pageNumber, \
baseDir,session,cachedWebfingers,personCache, \
nickname,domain,port,postJsonObject, \
avatarUrl,True,allowDeletion, \
httpPrefix,__version__,boxName, \
not isDM(postJsonObject), \
True,True,False,True)
updateRecentPostsCache(recentPostsCache,maxRecentPosts, \
postJsonObject,htmlStr)
def validInbox(baseDir: str,nickname: str,domain: str) -> bool:
"""Checks whether files were correctly saved to the inbox
@ -1603,7 +1630,8 @@ def inboxUpdateIndex(boxname: str,baseDir: str,handle: str,destinationFilename:
return False
def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
def inboxAfterCapabilities(recentPostsCache: {},maxRecentPosts: int, \
session,keyId: str,handle: str,messageJson: {}, \
baseDir: str,httpPrefix: str,sendThreads: [], \
postLog: [],cachedWebfingers: {},personCache: {}, \
queue: [],domain: str,port: int,useTor: bool, \
@ -1787,7 +1815,8 @@ def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
if debug:
print('DEBUG: saving inbox post as html to cache')
htmlCacheStartTime=time.time()
inboxStorePostToHtmlCache(translate,baseDir,httpPrefix, \
inboxStorePostToHtmlCache(recentPostsCache,maxRecentPosts, \
translate,baseDir,httpPrefix, \
session,cachedWebfingers,personCache, \
handle.split('@')[0],domain,port, \
postJsonObject,allowDeletion)
@ -1836,7 +1865,8 @@ def runInboxQueueWatchdog(projectVersion: str,httpd) -> None:
httpd.thrInboxQueue.start()
print('Restarting inbox queue...')
def runInboxQueue(projectVersion: str, \
def runInboxQueue(recentPostsCache: {},maxRecentPosts: int, \
projectVersion: str, \
baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \
cachedWebfingers: {},personCache: {},queue: [], \
domain: str,port: int,useTor: bool,federationList: [], \
@ -2175,7 +2205,8 @@ def runInboxQueue(projectVersion: str, \
# Here the capability id begins with the handle, so this could also
# be matched separately, but it's probably not necessary
if capsId in capabilityIdList:
inboxAfterCapabilities(session,keyId,handle, \
inboxAfterCapabilities(recentPostsCache,maxRecentPosts, \
session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \
@ -2194,7 +2225,8 @@ def runInboxQueue(projectVersion: str, \
pprint(queueJson['post'])
else:
if not ocapAlways:
inboxAfterCapabilities(session,keyId,handle, \
inboxAfterCapabilities(recentPostsCache,maxRecentPosts, \
session,keyId,handle, \
queueJson['post'], \
baseDir,httpPrefix, \
sendThreads,postLog, \

22
tests.py
View File

@ -67,6 +67,7 @@ from media import getAttachmentMediaType
from delete import sendDeleteViaServer
from inbox import validInbox
from inbox import validInboxFilenames
from inbox import updateRecentPostsCache
from content import addWebLinks
from content import replaceEmojiFromTags
from content import addHtmlTags
@ -232,7 +233,7 @@ def createServerAlice(path: str,domain: str,port: int,federationList: [], \
maxMentions=10
maxEmoji=10
print('Server running: Alice')
runDaemon(True,True,'en',__version__, \
runDaemon(5,True,True,'en',__version__, \
"instanceId",False,path,domain,port,port, \
httpPrefix,federationList,maxMentions,maxEmoji,False, \
noreply,nolike,nopics,noannounce,cw,ocapAlways, \
@ -288,7 +289,7 @@ def createServerBob(path: str,domain: str,port: int,federationList: [], \
maxMentions=10
maxEmoji=10
print('Server running: Bob')
runDaemon(True,True,'en',__version__, \
runDaemon(5,True,True,'en',__version__, \
"instanceId",False,path,domain,port,port, \
httpPrefix,federationList,maxMentions,maxEmoji,False, \
noreply,nolike,nopics,noannounce,cw,ocapAlways, \
@ -324,7 +325,7 @@ def createServerEve(path: str,domain: str,port: int,federationList: [], \
maxMentions=10
maxEmoji=10
print('Server running: Eve')
runDaemon(True,True,'en',__version__, \
runDaemon(5,True,True,'en',__version__, \
"instanceId",False,path,domain,port,port, \
httpPrefix,federationList,maxMentions,maxEmoji,False, \
noreply,nolike,nopics,noannounce,cw,ocapAlways, \
@ -1663,8 +1664,23 @@ def testTheme():
result=setCSSparam(css,'background-value','32px')
assert result=='--background-value: 32px; --foreground-value: 24px;'
def testRecentPostsCache():
print('testRecentPostsCache')
recentPostsCache={}
maxRecentPosts=3
htmlStr='<html></html>'
for i in range(5):
postJsonObject={
"id": "https://somesite.whatever/users/someuser/statuses/"+str(i)
}
updateRecentPostsCache(recentPostsCache,maxRecentPosts,postJsonObject,htmlStr)
assert len(recentPostsCache['index'])==maxRecentPosts
assert len(recentPostsCache['json'].items())==maxRecentPosts
assert len(recentPostsCache['html'].items())==maxRecentPosts
def runAllTests():
print('Running tests...')
testRecentPostsCache()
testTheme()
testSaveLoadJson()
testCommentJson()

94
webinterface.py
View File

@ -1876,7 +1876,13 @@ def saveIndividualPostAsHtmlToCache(baseDir: str,nickname: str,domain: str, \
except Exception as e:
print('ERROR: saving post to cache '+str(e))
return False
def preparePostFromHtmlCache(postHtml: str,boxName: str,pageNumber: int) -> str:
# if on the bookmarks timeline then remain there
if boxName=='tlbookmarks':
postHtml=postHtml.replace('?tl=inbox','?tl=tlbookmarks')
return postHtml.replace(';-999;',';'+str(pageNumber)+';').replace('?page=-999','?page='+str(pageNumber))
def individualPostAsHtml(iconsDir: str,translate: {}, \
pageNumber: int,baseDir: str, \
session,wfRequest: {},personCache: {}, \
@ -1908,10 +1914,7 @@ def individualPostAsHtml(iconsDir: str,translate: {}, \
loadIndividualPostAsHtmlFromCache(baseDir,nickname,domain, \
postJsonObject)
if postHtml:
# if on the bookmarks timeline then remain there
if boxName=='tlbookmarks':
postHtml=postHtml.replace('?tl=inbox','?tl=tlbookmarks')
return postHtml.replace(';-999;',';'+str(pageNumber)+';').replace('?page=-999','?page='+str(pageNumber))
return preparePostFromHtmlCache(postHtml,boxName,pageNumber)
# don't create new html within the bookmarks timeline
# it should already have been created for the inbox
@ -2365,7 +2368,8 @@ def isQuestion(postObjectJson: {}) -> bool:
return True
return False
def htmlTimeline(translate: {},pageNumber: int, \
def htmlTimeline(recentPostsCache: {}, \
translate: {},pageNumber: int, \
itemsPerPage: int,session,baseDir: str, \
wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,timelineJson: {}, \
@ -2576,16 +2580,26 @@ def htmlTimeline(translate: {},pageNumber: int, \
for item in timelineJson['orderedItems']:
if item['type']=='Create' or item['type']=='Announce':
#avatarUrl=getPersonAvatarUrl(baseDir,item['actor'],personCache)
currTlStr= \
individualPostAsHtml(iconsDir,translate,pageNumber, \
baseDir,session,wfRequest,personCache, \
nickname,domain,port,item,None,True, \
allowDeletion, \
httpPrefix,projectVersion,boxName, \
boxName!='dm', \
showIndividualPostIcons, \
manuallyApproveFollowers,False,True)
currTlStr=None
if recentPostsCache['index']:
postId=item['id'].replace('/activity','').replace('/','#')
if postId in recentPostsCache['index']:
if recentPostsCache['html'].get(postId):
currTlStr=recentPostsCache['html'][postId]
currTlStr= \
preparePostFromHtmlCache(currTlStr,boxName,pageNumber)
print('Post obtained from recent cache ('+str(len(recentPostsCache['index']))+'): '+postId)
if not currTlStr:
currTlStr= \
individualPostAsHtml(iconsDir,translate,pageNumber, \
baseDir,session,wfRequest,personCache, \
nickname,domain,port,item,None,True, \
allowDeletion, \
httpPrefix,projectVersion,boxName, \
boxName!='dm', \
showIndividualPostIcons, \
manuallyApproveFollowers,False,True)
if currTlStr:
itemCtr+=1
tlStr+=currTlStr
@ -2598,7 +2612,8 @@ def htmlTimeline(translate: {},pageNumber: int, \
tlStr+=htmlFooter()
return tlStr
def htmlShares(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlShares(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int, \
allowDeletion: bool, \
@ -2608,12 +2623,14 @@ def htmlShares(translate: {},pageNumber: int,itemsPerPage: int, \
manuallyApproveFollowers= \
followerApprovalActive(baseDir,nickname,domain)
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,None,'tlshares',allowDeletion, \
httpPrefix,projectVersion,manuallyApproveFollowers)
def htmlInbox(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlInbox(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,inboxJson: {}, \
allowDeletion: bool, \
@ -2623,12 +2640,14 @@ def htmlInbox(translate: {},pageNumber: int,itemsPerPage: int, \
manuallyApproveFollowers= \
followerApprovalActive(baseDir,nickname,domain)
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,inboxJson,'inbox',allowDeletion, \
httpPrefix,projectVersion,manuallyApproveFollowers)
def htmlBookmarks(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlBookmarks(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,bookmarksJson: {}, \
allowDeletion: bool, \
@ -2638,60 +2657,70 @@ def htmlBookmarks(translate: {},pageNumber: int,itemsPerPage: int, \
manuallyApproveFollowers= \
followerApprovalActive(baseDir,nickname,domain)
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,bookmarksJson,'tlbookmarks',allowDeletion, \
httpPrefix,projectVersion,manuallyApproveFollowers)
def htmlInboxDMs(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlInboxDMs(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,inboxJson: {}, \
allowDeletion: bool, \
httpPrefix: str,projectVersion: str) -> str:
"""Show the DM timeline as html
"""
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,inboxJson,'dm',allowDeletion, \
httpPrefix,projectVersion,False)
def htmlInboxReplies(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlInboxReplies(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,inboxJson: {}, \
allowDeletion: bool, \
httpPrefix: str,projectVersion: str) -> str:
"""Show the replies timeline as html
"""
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,inboxJson,'tlreplies',allowDeletion, \
httpPrefix,projectVersion,False)
def htmlInboxMedia(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlInboxMedia(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,inboxJson: {}, \
allowDeletion: bool, \
httpPrefix: str,projectVersion: str) -> str:
"""Show the media timeline as html
"""
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,inboxJson,'tlmedia',allowDeletion, \
httpPrefix,projectVersion,False)
def htmlModeration(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlModeration(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,inboxJson: {}, \
allowDeletion: bool, \
httpPrefix: str,projectVersion: str) -> str:
"""Show the moderation feed as html
"""
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,inboxJson,'moderation',allowDeletion, \
httpPrefix,projectVersion,True)
def htmlOutbox(translate: {},pageNumber: int,itemsPerPage: int, \
def htmlOutbox(recentPostsCache: {}, \
translate: {},pageNumber: int,itemsPerPage: int, \
session,baseDir: str,wfRequest: {},personCache: {}, \
nickname: str,domain: str,port: int,outboxJson: {}, \
allowDeletion: bool,
@ -2700,7 +2729,8 @@ def htmlOutbox(translate: {},pageNumber: int,itemsPerPage: int, \
"""
manuallyApproveFollowers= \
followerApprovalActive(baseDir,nickname,domain)
return htmlTimeline(translate,pageNumber, \
return htmlTimeline(recentPostsCache, \
translate,pageNumber, \
itemsPerPage,session,baseDir,wfRequest,personCache, \
nickname,domain,port,outboxJson,'outbox',allowDeletion, \
httpPrefix,projectVersion,manuallyApproveFollowers)