Merge branch 'main' of ssh://code.freedombone.net:2222/bashrc/epicyon into main

main
Bob Mottram 2020-12-13 22:45:39 +00:00
commit 405eb1337f
22 changed files with 287 additions and 18 deletions

View File

@ -622,6 +622,7 @@ def getBlogIndexesForAccounts(baseDir: str) -> {}:
blogsIndex = accountDir + '/tlblogs.index' blogsIndex = accountDir + '/tlblogs.index'
if os.path.isfile(blogsIndex): if os.path.isfile(blogsIndex):
blogIndexes[acct] = blogsIndex blogIndexes[acct] = blogsIndex
break
return blogIndexes return blogIndexes
@ -639,6 +640,7 @@ def noOfBlogAccounts(baseDir: str) -> int:
blogsIndex = accountDir + '/tlblogs.index' blogsIndex = accountDir + '/tlblogs.index'
if os.path.isfile(blogsIndex): if os.path.isfile(blogsIndex):
ctr += 1 ctr += 1
break
return ctr return ctr
@ -655,6 +657,7 @@ def singleBlogAccountNickname(baseDir: str) -> str:
blogsIndex = accountDir + '/tlblogs.index' blogsIndex = accountDir + '/tlblogs.index'
if os.path.isfile(blogsIndex): if os.path.isfile(blogsIndex):
return acct.split('@')[0] return acct.split('@')[0]
break
return None return None
@ -698,6 +701,7 @@ def htmlBlogView(authorized: bool,
httpPrefix + '://' + domainFull + '/blog/' + \ httpPrefix + '://' + domainFull + '/blog/' + \
acct.split('@')[0] + '">' + acct + '</a>' acct.split('@')[0] + '">' + acct + '</a>'
blogStr += '</p>' blogStr += '</p>'
break
return blogStr + htmlFooter() return blogStr + htmlFooter()

View File

@ -508,6 +508,15 @@ def addEmoji(baseDir: str, wordStr: str,
return True return True
def tagExists(tagType: str, tagName: str, tags: {}) -> bool:
"""Returns true if a tag exists in the given dict
"""
for tag in tags:
if tag['name'] == tagName and tag['type'] == tagType:
return True
return False
def addMention(wordStr: str, httpPrefix: str, following: str, def addMention(wordStr: str, httpPrefix: str, following: str,
replaceMentions: {}, recipients: [], tags: {}) -> bool: replaceMentions: {}, recipients: [], tags: {}) -> bool:
"""Detects mentions and adds them to the replacements dict and """Detects mentions and adds them to the replacements dict and

View File

@ -89,6 +89,7 @@ from inbox import getPersonPubKey
from follow import getFollowingFeed from follow import getFollowingFeed
from follow import sendFollowRequest from follow import sendFollowRequest
from follow import unfollowPerson from follow import unfollowPerson
from follow import createInitialLastSeen
from auth import authorize from auth import authorize
from auth import createPassword from auth import createPassword
from auth import createBasicAuthHeader from auth import createBasicAuthHeader
@ -4788,6 +4789,7 @@ class PubServer(BaseHTTPRequestHandler):
port, port,
maxPostsInRSSFeed, 1, maxPostsInRSSFeed, 1,
False) False)
break
if msg: if msg:
msg = rss2Header(httpPrefix, msg = rss2Header(httpPrefix,
'news', domainFull, 'news', domainFull,
@ -4987,7 +4989,8 @@ class PubServer(BaseHTTPRequestHandler):
ssbAddress, blogAddress, ssbAddress, blogAddress,
toxAddress, jamiAddress, toxAddress, jamiAddress,
PGPpubKey, PGPfingerprint, PGPpubKey, PGPfingerprint,
emailAddress).encode('utf-8') emailAddress,
self.server.dormantMonths).encode('utf-8')
self._set_headers('text/html', len(msg), self._set_headers('text/html', len(msg),
cookie, callingDomain) cookie, callingDomain)
self._write(msg) self._write(msg)
@ -6491,6 +6494,7 @@ class PubServer(BaseHTTPRequestHandler):
YTReplacementDomain, YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
actorJson['roles'], actorJson['roles'],
None, None) None, None)
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
@ -6570,6 +6574,7 @@ class PubServer(BaseHTTPRequestHandler):
YTReplacementDomain, YTReplacementDomain,
showPublishedDateOnly, showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
actorJson['skills'], actorJson['skills'],
None, None) None, None)
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
@ -8258,6 +8263,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain, self.server.YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
shares, shares,
pageNumber, sharesPerPage) pageNumber, sharesPerPage)
msg = msg.encode('utf-8') msg = msg.encode('utf-8')
@ -8349,6 +8355,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain, self.server.YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
following, following,
pageNumber, pageNumber,
followsPerPage).encode('utf-8') followsPerPage).encode('utf-8')
@ -8440,6 +8447,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain, self.server.YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
followers, followers,
pageNumber, pageNumber,
followsPerPage).encode('utf-8') followsPerPage).encode('utf-8')
@ -8506,6 +8514,7 @@ class PubServer(BaseHTTPRequestHandler):
self.server.YTReplacementDomain, self.server.YTReplacementDomain,
self.server.showPublishedDateOnly, self.server.showPublishedDateOnly,
self.server.newswire, self.server.newswire,
self.server.dormantMonths,
None, None).encode('utf-8') None, None).encode('utf-8')
self._set_headers('text/html', len(msg), self._set_headers('text/html', len(msg),
cookie, callingDomain) cookie, callingDomain)
@ -12125,6 +12134,7 @@ class PubServer(BaseHTTPRequestHandler):
contentJson = loadJson(deviceFilename) contentJson = loadJson(deviceFilename)
if contentJson: if contentJson:
devicesList.append(contentJson) devicesList.append(contentJson)
break
# return the list of devices for this handle # return the list of devices for this handle
msg = \ msg = \
json.dumps(devicesList, json.dumps(devicesList,
@ -12924,9 +12934,11 @@ def loadTokens(baseDir: str, tokensDict: {}, tokensLookup: {}) -> None:
continue continue
tokensDict[nickname] = token tokensDict[nickname] = token
tokensLookup[token] = nickname tokensLookup[token] = nickname
break
def runDaemon(maxNewswirePosts: int, def runDaemon(dormantMonths: int,
maxNewswirePosts: int,
allowLocalNetworkAccess: bool, allowLocalNetworkAccess: bool,
maxFeedItemSizeKb: int, maxFeedItemSizeKb: int,
publishButtonAtTop: bool, publishButtonAtTop: bool,
@ -13120,6 +13132,10 @@ def runDaemon(maxNewswirePosts: int,
# maximum size of a hashtag category, in K # maximum size of a hashtag category, in K
httpd.maxCategoriesFeedItemSizeKb = 1024 httpd.maxCategoriesFeedItemSizeKb = 1024
# how many months does a followed account need to be unseen
# for it to be considered dormant?
httpd.dormantMonths = dormantMonths
if registration == 'open': if registration == 'open':
httpd.registration = True httpd.registration = True
else: else:
@ -13249,6 +13265,8 @@ def runDaemon(maxNewswirePosts: int,
httpd.iconsCache = {} httpd.iconsCache = {}
httpd.fontsCache = {} httpd.fontsCache = {}
createInitialLastSeen(baseDir, httpPrefix)
print('Creating inbox queue') print('Creating inbox queue')
httpd.thrInboxQueue = \ httpd.thrInboxQueue = \
threadWithTrace(target=runInboxQueue, threadWithTrace(target=runInboxQueue,

View File

@ -321,6 +321,7 @@ def removeOldHashtags(baseDir: str, maxMonths: int) -> str:
# check of the file is too old # check of the file is too old
if fileDaysSinceEpoch < maxDaysSinceEpoch: if fileDaysSinceEpoch < maxDaysSinceEpoch:
removeHashtags.append(tagsFilename) removeHashtags.append(tagsFilename)
break
for removeFilename in removeHashtags: for removeFilename in removeHashtags:
try: try:

View File

@ -152,6 +152,7 @@ def E2EEdevicesCollection(baseDir: str, nickname: str, domain: str,
devJson = loadJson(deviceFilename) devJson = loadJson(deviceFilename)
if devJson: if devJson:
deviceList.append(devJson) deviceList.append(devJson)
break
devicesDict = { devicesDict = {
'id': personId + '/collections/devices', 'id': personId + '/collections/devices',

View File

@ -116,6 +116,11 @@ parser.add_argument('--postsPerSource',
dest='maxNewswirePostsPerSource', type=int, dest='maxNewswirePostsPerSource', type=int,
default=4, default=4,
help='Maximum newswire posts per feed or account') help='Maximum newswire posts per feed or account')
parser.add_argument('--dormantMonths',
dest='dormantMonths', type=int,
default=3,
help='How many months does a followed account need to ' +
'be unseen for before being considered dormant')
parser.add_argument('--maxNewswirePosts', parser.add_argument('--maxNewswirePosts',
dest='maxNewswirePosts', type=int, dest='maxNewswirePosts', type=int,
default=20, default=20,
@ -2032,6 +2037,11 @@ maxFeedItemSizeKb = \
if maxFeedItemSizeKb is not None: if maxFeedItemSizeKb is not None:
args.maxFeedItemSizeKb = int(maxFeedItemSizeKb) args.maxFeedItemSizeKb = int(maxFeedItemSizeKb)
dormantMonths = \
getConfigParam(baseDir, 'dormantMonths')
if dormantMonths is not None:
args.dormantMonths = int(dormantMonths)
allowNewsFollowers = \ allowNewsFollowers = \
getConfigParam(baseDir, 'allowNewsFollowers') getConfigParam(baseDir, 'allowNewsFollowers')
if allowNewsFollowers is not None: if allowNewsFollowers is not None:
@ -2080,7 +2090,8 @@ if setTheme(baseDir, themeName, domain, args.allowLocalNetworkAccess):
print('Theme set to ' + themeName) print('Theme set to ' + themeName)
if __name__ == "__main__": if __name__ == "__main__":
runDaemon(args.maxNewswirePosts, runDaemon(args.dormantMonths,
args.maxNewswirePosts,
args.allowLocalNetworkAccess, args.allowLocalNetworkAccess,
args.maxFeedItemSizeKb, args.maxFeedItemSizeKb,
args.publishButtonAtTop, args.publishButtonAtTop,

View File

@ -27,6 +27,43 @@ from auth import createBasicAuthHeader
from session import postJson from session import postJson
def createInitialLastSeen(baseDir: str, httpPrefix: str) -> None:
"""Creates initial lastseen files for all follows
"""
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
for acct in dirs:
if '@' not in acct:
continue
if 'inbox@' in acct or 'news@' in acct:
continue
accountDir = os.path.join(baseDir + '/accounts', acct)
followingFilename = accountDir + '/following.txt'
if not os.path.isfile(followingFilename):
continue
lastSeenDir = accountDir + '/lastseen'
if not os.path.isdir(lastSeenDir):
os.mkdir(lastSeenDir)
with open(followingFilename, 'r') as fp:
followingHandles = fp.readlines()
for handle in followingHandles:
if '#' in handle:
continue
if '@' not in handle:
continue
handle = handle.replace('\n', '')
nickname = handle.split('@')[0]
domain = handle.split('@')[1]
actor = \
httpPrefix + '://' + domain + '/users/' + nickname
lastSeenFilename = \
lastSeenDir + '/' + actor.replace('/', '#') + '.txt'
print('lastSeenFilename: ' + lastSeenFilename)
if not os.path.isfile(lastSeenFilename):
with open(lastSeenFilename, 'w+') as fp:
fp.write(str(100))
break
def preApprovedFollower(baseDir: str, def preApprovedFollower(baseDir: str,
nickname: str, domain: str, nickname: str, domain: str,
approveHandle: str, approveHandle: str,
@ -1167,6 +1204,7 @@ def getFollowersOfActor(baseDir: str, actor: str, debug: bool) -> {}:
print('DEBUG: ' + account + print('DEBUG: ' + account +
' follows ' + actorHandle) ' follows ' + actorHandle)
recipientsDict[account] = None recipientsDict[account] = None
break
return recipientsDict return recipientsDict

View File

@ -67,6 +67,7 @@ from followingCalendar import receivingCalendarEvents
from content import dangerousMarkup from content import dangerousMarkup
from happening import saveEventPost from happening import saveEventPost
from delete import removeOldHashtags from delete import removeOldHashtags
from follow import isFollowingActor
def guessHashtagCategory(tagName: str, hashtagCategories: {}) -> str: def guessHashtagCategory(tagName: str, hashtagCategories: {}) -> str:
@ -199,6 +200,7 @@ def validInbox(baseDir: str, nickname: str, domain: str) -> bool:
if 'postNickname' in open(filename).read(): if 'postNickname' in open(filename).read():
print('queue file incorrectly saved to ' + filename) print('queue file incorrectly saved to ' + filename)
return False return False
break
return True return True
@ -223,6 +225,7 @@ def validInboxFilenames(baseDir: str, nickname: str, domain: str,
print('Expected: ' + expectedStr) print('Expected: ' + expectedStr)
print('Invalid filename: ' + filename) print('Invalid filename: ' + filename)
return False return False
break
return True return True
@ -2066,6 +2069,38 @@ def inboxUpdateIndex(boxname: str, baseDir: str, handle: str,
return False return False
def updateLastSeen(baseDir: str, handle: str, actor: str) -> None:
"""Updates the time when the given handle last saw the given actor
This can later be used to indicate if accounts are dormant/abandoned/moved
"""
if '@' not in handle:
return
nickname = handle.split('@')[0]
domain = handle.split('@')[1]
if ':' in domain:
domain = domain.split(':')[0]
accountPath = baseDir + '/accounts/' + nickname + '@' + domain
if not os.path.isdir(accountPath):
return
if not isFollowingActor(baseDir, nickname, domain, actor):
return
lastSeenPath = accountPath + '/lastseen'
if not os.path.isdir(lastSeenPath):
os.mkdir(lastSeenPath)
lastSeenFilename = lastSeenPath + '/' + actor.replace('/', '#') + '.txt'
currTime = datetime.datetime.utcnow()
daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days
# has the value changed?
if os.path.isfile(lastSeenFilename):
with open(lastSeenFilename, 'r') as lastSeenFile:
daysSinceEpochFile = lastSeenFile.read()
if int(daysSinceEpochFile) == daysSinceEpoch:
# value hasn't changed, so we can save writing anything to file
return
with open(lastSeenFilename, 'w+') as lastSeenFile:
lastSeenFile.write(str(daysSinceEpoch))
def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int, def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
session, keyId: str, handle: str, messageJson: {}, session, keyId: str, handle: str, messageJson: {},
baseDir: str, httpPrefix: str, sendThreads: [], baseDir: str, httpPrefix: str, sendThreads: [],
@ -2086,6 +2121,8 @@ def inboxAfterInitial(recentPostsCache: {}, maxRecentPosts: int,
if '#' in actor: if '#' in actor:
actor = keyId.split('#')[0] actor = keyId.split('#')[0]
updateLastSeen(baseDir, handle, actor)
isGroup = groupHandle(baseDir, handle) isGroup = groupHandle(baseDir, handle)
if receiveLike(recentPostsCache, if receiveLike(recentPostsCache,
@ -2436,6 +2473,7 @@ def clearQueueItems(baseDir: str, queue: []) -> None:
ctr += 1 ctr += 1
except BaseException: except BaseException:
pass pass
break
if ctr > 0: if ctr > 0:
print('Removed ' + str(ctr) + ' inbox queue items') print('Removed ' + str(ctr) + ' inbox queue items')
@ -2452,6 +2490,7 @@ def restoreQueueItems(baseDir: str, queue: []) -> None:
for queuesubdir, queuedirs, queuefiles in os.walk(queueDir): for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
for qfile in queuefiles: for qfile in queuefiles:
queue.append(os.path.join(queueDir, qfile)) queue.append(os.path.join(queueDir, qfile))
break
if len(queue) > 0: if len(queue) > 0:
print('Restored ' + str(len(queue)) + ' inbox queue items') print('Restored ' + str(len(queue)) + ' inbox queue items')

View File

@ -221,3 +221,4 @@ def archiveMedia(baseDir: str, archiveDirectory: str, maxWeeks=4) -> None:
else: else:
# archive to /dev/null # archive to /dev/null
rmtree(os.path.join(baseDir + '/media', weekDir)) rmtree(os.path.join(baseDir + '/media', weekDir))
break

View File

@ -51,3 +51,4 @@ def migrateAccount(baseDir: str, oldHandle: str, newHandle: str) -> None:
migrateFollows(followFilename, oldHandle, newHandle) migrateFollows(followFilename, oldHandle, newHandle)
followFilename = accountDir + '/followers.txt' followFilename = accountDir + '/followers.txt'
migrateFollows(followFilename, oldHandle, newHandle) migrateFollows(followFilename, oldHandle, newHandle)
break

View File

@ -760,6 +760,7 @@ def addBlogsToNewswire(baseDir: str, domain: str, newswire: {},
addAccountBlogsToNewswire(baseDir, nickname, domain, addAccountBlogsToNewswire(baseDir, nickname, domain,
newswire, maxBlogsPerAccount, newswire, maxBlogsPerAccount,
blogsIndex, maxTags) blogsIndex, maxTags)
break
# sort the moderation dict into chronological order, latest first # sort the moderation dict into chronological order, latest first
sortedModerationDict = \ sortedModerationDict = \

View File

@ -52,6 +52,7 @@ from utils import votesOnNewswireItem
from utils import removeHtml from utils import removeHtml
from media import attachMedia from media import attachMedia
from media import replaceYouTube from media import replaceYouTube
from content import tagExists
from content import removeLongWords from content import removeLongWords
from content import addHtmlTags from content import addHtmlTags
from content import replaceEmojiFromTags from content import replaceEmojiFromTags
@ -801,7 +802,8 @@ def createPostBase(baseDir: str, nickname: str, domain: str, port: int,
isPublic = True isPublic = True
break break
for tagName, tag in hashtagsDict.items(): for tagName, tag in hashtagsDict.items():
tags.append(tag) if not tagExists(tag['type'], tag['name'], tags):
tags.append(tag)
if isPublic: if isPublic:
updateHashtagsIndex(baseDir, tag, newPostId) updateHashtagsIndex(baseDir, tag, newPostId)
print('Content tags: ' + str(tags)) print('Content tags: ' + str(tags))
@ -1010,6 +1012,16 @@ def createPostBase(baseDir: str, nickname: str, domain: str, port: int,
if newPost.get('object'): if newPost.get('object'):
newPost['object']['cc'] = [ccUrl] newPost['object']['cc'] = [ccUrl]
# if this is a public post then include any mentions in cc
toCC = newPost['object']['cc']
if len(toRecipients) == 1:
if toRecipients[0].endswith('#Public') and \
ccUrl.endswith('/followers'):
for tag in tags:
if tag['type'] == 'Mention':
if tag['href'] not in toCC:
toCC.append(tag['href'])
# if this is a moderation report then add a status # if this is a moderation report then add a status
if isModerationReport: if isModerationReport:
# add status # add status
@ -3183,6 +3195,7 @@ def archivePosts(baseDir: str, httpPrefix: str, archiveDir: str,
archivePostsForPerson(httpPrefix, nickname, domain, baseDir, archivePostsForPerson(httpPrefix, nickname, domain, baseDir,
'outbox', archiveSubdir, 'outbox', archiveSubdir,
recentPostsCache, maxPostsInBox) recentPostsCache, maxPostsInBox)
break
def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str, def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,

View File

@ -146,6 +146,7 @@ def runPostSchedule(baseDir: str, httpd, maxScheduledPosts: int):
if not os.path.isfile(scheduleIndexFilename): if not os.path.isfile(scheduleIndexFilename):
continue continue
updatePostSchedule(baseDir, account, httpd, maxScheduledPosts) updatePostSchedule(baseDir, account, httpd, maxScheduledPosts)
break
def runPostScheduleWatchdog(projectVersion: str, httpd) -> None: def runPostScheduleWatchdog(projectVersion: str, httpd) -> None:

View File

@ -167,6 +167,7 @@ def addShare(baseDir: str,
'/users/' + nickname + '/tlshares') '/users/' + nickname + '/tlshares')
except BaseException: except BaseException:
pass pass
break
def expireShares(baseDir: str) -> None: def expireShares(baseDir: str) -> None:
@ -179,6 +180,7 @@ def expireShares(baseDir: str) -> None:
nickname = account.split('@')[0] nickname = account.split('@')[0]
domain = account.split('@')[1] domain = account.split('@')[1]
expireSharesForAccount(baseDir, nickname, domain) expireSharesForAccount(baseDir, nickname, domain)
break
def expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None: def expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None:

View File

@ -20,6 +20,7 @@ from cache import getPersonFromCache
from threads import threadWithTrace from threads import threadWithTrace
from daemon import runDaemon from daemon import runDaemon
from session import createSession from session import createSession
from posts import getMentionedPeople
from posts import validContentWarning from posts import validContentWarning
from posts import deleteAllPosts from posts import deleteAllPosts
from posts import createPublicPost from posts import createPublicPost
@ -75,6 +76,7 @@ from inbox import guessHashtagCategory
from content import htmlReplaceEmailQuote from content import htmlReplaceEmailQuote
from content import htmlReplaceQuoteMarks from content import htmlReplaceQuoteMarks
from content import dangerousMarkup from content import dangerousMarkup
from content import dangerousCSS
from content import addWebLinks from content import addWebLinks
from content import replaceEmojiFromTags from content import replaceEmojiFromTags
from content import addHtmlTags from content import addHtmlTags
@ -296,8 +298,10 @@ def createServerAlice(path: str, domain: str, port: int,
i2pDomain = None i2pDomain = None
allowLocalNetworkAccess = True allowLocalNetworkAccess = True
maxNewswirePosts = 20 maxNewswirePosts = 20
dormantMonths = 3
print('Server running: Alice') print('Server running: Alice')
runDaemon(maxNewswirePosts, allowLocalNetworkAccess, runDaemon(dormantMonths, maxNewswirePosts,
allowLocalNetworkAccess,
2048, False, True, False, False, True, 10, False, 2048, False, True, False, False, True, 10, False,
0, 100, 1024, 5, False, 0, 100, 1024, 5, False,
0, False, 1, False, False, False, 0, False, 1, False, False, False,
@ -364,8 +368,10 @@ def createServerBob(path: str, domain: str, port: int,
i2pDomain = None i2pDomain = None
allowLocalNetworkAccess = True allowLocalNetworkAccess = True
maxNewswirePosts = 20 maxNewswirePosts = 20
dormantMonths = 3
print('Server running: Bob') print('Server running: Bob')
runDaemon(maxNewswirePosts, allowLocalNetworkAccess, runDaemon(dormantMonths, maxNewswirePosts,
allowLocalNetworkAccess,
2048, False, True, False, False, True, 10, False, 2048, False, True, False, False, True, 10, False,
0, 100, 1024, 5, False, 0, 0, 100, 1024, 5, False, 0,
False, 1, False, False, False, False, 1, False, False, False,
@ -406,8 +412,10 @@ def createServerEve(path: str, domain: str, port: int, federationList: [],
i2pDomain = None i2pDomain = None
allowLocalNetworkAccess = True allowLocalNetworkAccess = True
maxNewswirePosts = 20 maxNewswirePosts = 20
dormantMonths = 3
print('Server running: Eve') print('Server running: Eve')
runDaemon(maxNewswirePosts, allowLocalNetworkAccess, runDaemon(dormantMonths, maxNewswirePosts,
allowLocalNetworkAccess,
2048, False, True, False, False, True, 10, False, 2048, False, True, False, False, True, 10, False,
0, 100, 1024, 5, False, 0, 0, 100, 1024, 5, False, 0,
False, 1, False, False, False, False, 1, False, False, False,
@ -1683,7 +1691,7 @@ def testWebLinks():
'This post has a web links https://somesite.net\n\nAnd some other text' 'This post has a web links https://somesite.net\n\nAnd some other text'
linkedText = addWebLinks(exampleText) linkedText = addWebLinks(exampleText)
assert \ assert \
'<a href="https://somesite.net" rel="nofollow noopener"' + \ '<a href="https://somesite.net" rel="nofollow noopener noreferrer"' + \
' target="_blank"><span class="invisible">https://' + \ ' target="_blank"><span class="invisible">https://' + \
'</span><span class="ellipsis">somesite.net</span></a' in linkedText '</span><span class="ellipsis">somesite.net</span></a' in linkedText
@ -1978,6 +1986,17 @@ def testRemoveHtml():
assert(removeHtml(testStr) == 'This string has html.') assert(removeHtml(testStr) == 'This string has html.')
def testDangerousCSS():
print('testDangerousCSS')
baseDir = os.getcwd()
for subdir, dirs, files in os.walk(baseDir):
for f in files:
if not f.endswith('.css'):
continue
assert not dangerousCSS(baseDir + '/' + f, False)
break
def testDangerousMarkup(): def testDangerousMarkup():
print('testDangerousMarkup') print('testDangerousMarkup')
allowLocalNetworkAccess = False allowLocalNetworkAccess = False
@ -2461,8 +2480,56 @@ def testGuessHashtagCategory() -> None:
assert guess == "bar" assert guess == "bar"
def testGetMentionedPeople() -> None:
print('testGetMentionedPeople')
baseDir = os.getcwd()
content = "@dragon@cave.site @bat@cave.site This is a test."
actors = getMentionedPeople(baseDir, 'https',
content,
'mydomain', False)
assert actors
assert len(actors) == 2
assert actors[0] == "https://cave.site/users/dragon"
assert actors[1] == "https://cave.site/users/bat"
def testReplyToPublicPost() -> None:
baseDir = os.getcwd()
nickname = 'test7492362'
domain = 'other.site'
port = 443
httpPrefix = 'https'
postId = httpPrefix + '://rat.site/users/ninjarodent/statuses/63746173435'
reply = \
createPublicPost(baseDir, nickname, domain, port, httpPrefix,
"@ninjarodent@rat.site This is a test.",
False, False, False, True,
None, None, False, postId)
# print(str(reply))
assert reply['object']['content'] == \
'<p><span class=\"h-card\">' + \
'<a href=\"https://rat.site/@ninjarodent\" ' + \
'class=\"u-url mention\">@<span>ninjarodent</span>' + \
'</a></span> This is a test.</p>'
assert reply['object']['tag'][0]['type'] == 'Mention'
assert reply['object']['tag'][0]['name'] == '@ninjarodent@rat.site'
assert reply['object']['tag'][0]['href'] == \
'https://rat.site/users/ninjarodent'
assert len(reply['object']['to']) == 1
assert reply['object']['to'][0].endswith('#Public')
assert len(reply['object']['cc']) >= 1
assert reply['object']['cc'][0].endswith(nickname + '/followers')
assert len(reply['object']['tag']) == 1
assert len(reply['object']['cc']) == 2
assert reply['object']['cc'][1] == \
httpPrefix + '://rat.site/users/ninjarodent'
def runAllTests(): def runAllTests():
print('Running tests...') print('Running tests...')
testReplyToPublicPost()
testGetMentionedPeople()
testGuessHashtagCategory() testGuessHashtagCategory()
testValidNickname() testValidNickname()
testParseFeedDate() testParseFeedDate()
@ -2477,6 +2544,7 @@ def runAllTests():
testRemoveIdEnding() testRemoveIdEnding()
testJsonPostAllowsComments() testJsonPostAllowsComments()
runHtmlReplaceQuoteMarks() runHtmlReplaceQuoteMarks()
testDangerousCSS()
testDangerousMarkup() testDangerousMarkup()
testRemoveHtml() testRemoveHtml()
testSiteIsActive() testSiteIsActive()

View File

@ -557,6 +557,7 @@ def setThemeImages(baseDir: str, name: str) -> None:
os.remove(accountDir + '/right_col_image.png') os.remove(accountDir + '/right_col_image.png')
except BaseException: except BaseException:
pass pass
break
def setNewsAvatar(baseDir: str, name: str, def setNewsAvatar(baseDir: str, name: str,

View File

@ -19,6 +19,30 @@ from calendar import monthrange
from followingCalendar import addPersonToCalendar from followingCalendar import addPersonToCalendar
def isDormant(baseDir: str, nickname: str, domain: str, actor: str,
dormantMonths=3) -> bool:
"""Is the given followed actor dormant, from the standpoint
of the given account
"""
lastSeenFilename = \
baseDir + '/accounts/' + nickname + '@' + domain + \
'/lastseen/' + actor.replace('/', '#') + '.txt'
if not os.path.isfile(lastSeenFilename):
return False
with open(lastSeenFilename, 'r') as lastSeenFile:
daysSinceEpochStr = lastSeenFile.read()
daysSinceEpoch = int(daysSinceEpochStr)
currTime = datetime.datetime.utcnow()
currDaysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days
timeDiffMonths = \
int((currDaysSinceEpoch - daysSinceEpoch) / 30)
if timeDiffMonths >= dormantMonths:
return True
return False
def getHashtagCategory(baseDir: str, hashtag: str) -> str: def getHashtagCategory(baseDir: str, hashtag: str) -> str:
"""Returns the category for the hashtag """Returns the category for the hashtag
""" """
@ -86,6 +110,7 @@ def getHashtagCategories(baseDir: str, recent=False, category=None) -> None:
else: else:
if hashtag not in hashtagCategories[categoryStr]: if hashtag not in hashtagCategories[categoryStr]:
hashtagCategories[categoryStr].append(hashtag) hashtagCategories[categoryStr].append(hashtag)
break
return hashtagCategories return hashtagCategories
@ -383,6 +408,7 @@ def getFollowersOfPerson(baseDir: str,
if account not in followers: if account not in followers:
followers.append(account) followers.append(account)
break break
break
return followers return followers
@ -908,6 +934,7 @@ def clearFromPostCaches(baseDir: str, recentPostsCache: {},
if recentPostsCache.get('html'): if recentPostsCache.get('html'):
if recentPostsCache['html'].get(postId): if recentPostsCache['html'].get(postId):
del recentPostsCache['html'][postId] del recentPostsCache['html'][postId]
break
def locatePost(baseDir: str, nickname: str, domain: str, def locatePost(baseDir: str, nickname: str, domain: str,
@ -1171,6 +1198,7 @@ def noOfAccounts(baseDir: str) -> bool:
if '@' in account: if '@' in account:
if not account.startswith('inbox@'): if not account.startswith('inbox@'):
accountCtr += 1 accountCtr += 1
break
return accountCtr return accountCtr
@ -1193,6 +1221,7 @@ def noOfActiveAccountsMonthly(baseDir: str, months: int) -> bool:
timeDiff = (currTime - int(lastUsed)) timeDiff = (currTime - int(lastUsed))
if timeDiff < monthSeconds: if timeDiff < monthSeconds:
accountCtr += 1 accountCtr += 1
break
return accountCtr return accountCtr
@ -1469,6 +1498,7 @@ def searchBoxPosts(baseDir: str, nickname: str, domain: str,
res.append(filePath) res.append(filePath)
if len(res) >= maxResults: if len(res) >= maxResults:
return res return res
break
return res return res

View File

@ -197,6 +197,7 @@ def htmlHashTagSwarm(baseDir: str, actor: str, translate: {}) -> str:
if categoryStr not in categorySwarm: if categoryStr not in categorySwarm:
categorySwarm.append(categoryStr) categorySwarm.append(categoryStr)
break break
break
if not tagSwarm: if not tagSwarm:
return '' return ''

View File

@ -11,6 +11,7 @@ from shutil import copyfile
from petnames import getPetName from petnames import getPetName
from person import isPersonSnoozed from person import isPersonSnoozed
from posts import isModerator from posts import isModerator
from utils import isDormant
from utils import removeHtml from utils import removeHtml
from utils import getDomainFromActor from utils import getDomainFromActor
from utils import getNicknameFromActor from utils import getNicknameFromActor
@ -39,7 +40,8 @@ def htmlPersonOptions(defaultTimeline: str,
jamiAddress: str, jamiAddress: str,
PGPpubKey: str, PGPpubKey: str,
PGPfingerprint: str, PGPfingerprint: str,
emailAddress) -> str: emailAddress: str,
dormantMonths: int) -> str:
"""Show options for a person: view/follow/block/report """Show options for a person: view/follow/block/report
""" """
optionsDomain, optionsPort = getDomainFromActor(optionsActor) optionsDomain, optionsPort = getDomainFromActor(optionsActor)
@ -53,6 +55,7 @@ def htmlPersonOptions(defaultTimeline: str,
copyfile(baseDir + '/accounts/options-background.jpg', copyfile(baseDir + '/accounts/options-background.jpg',
baseDir + '/accounts/options-background.jpg') baseDir + '/accounts/options-background.jpg')
dormant = False
followStr = 'Follow' followStr = 'Follow'
blockStr = 'Block' blockStr = 'Block'
nickname = None nickname = None
@ -66,6 +69,9 @@ def htmlPersonOptions(defaultTimeline: str,
followerDomain, followerPort = getDomainFromActor(optionsActor) followerDomain, followerPort = getDomainFromActor(optionsActor)
if isFollowingActor(baseDir, nickname, domain, optionsActor): if isFollowingActor(baseDir, nickname, domain, optionsActor):
followStr = 'Unfollow' followStr = 'Unfollow'
dormant = \
isDormant(baseDir, nickname, domain, optionsActor,
dormantMonths)
optionsNickname = getNicknameFromActor(optionsActor) optionsNickname = getNicknameFromActor(optionsActor)
optionsDomainFull = optionsDomain optionsDomainFull = optionsDomain
@ -107,9 +113,12 @@ def htmlPersonOptions(defaultTimeline: str,
optionsStr += ' <img loading="lazy" src="' + optionsProfileUrl + \ optionsStr += ' <img loading="lazy" src="' + optionsProfileUrl + \
'" ' + getBrokenLinkSubstitute() + '/></a>\n' '" ' + getBrokenLinkSubstitute() + '/></a>\n'
handle = getNicknameFromActor(optionsActor) + '@' + optionsDomain handle = getNicknameFromActor(optionsActor) + '@' + optionsDomain
handleShown = handle
if dormant:
handleShown += ' 💤'
optionsStr += \ optionsStr += \
' <p class="optionsText">' + translate['Options for'] + \ ' <p class="optionsText">' + translate['Options for'] + \
' @' + handle + '</p>\n' ' @' + handleShown + '</p>\n'
if emailAddress: if emailAddress:
optionsStr += \ optionsStr += \
'<p class="imText">' + translate['Email'] + \ '<p class="imText">' + translate['Email'] + \

View File

@ -8,6 +8,7 @@ __status__ = "Production"
import os import os
from pprint import pprint from pprint import pprint
from utils import isDormant
from utils import getNicknameFromActor from utils import getNicknameFromActor
from utils import getDomainFromActor from utils import getDomainFromActor
from utils import isSystemAccount from utils import isSystemAccount
@ -364,8 +365,9 @@ def htmlProfile(rssIconAtTop: bool,
session, wfRequest: {}, personCache: {}, session, wfRequest: {}, personCache: {},
YTReplacementDomain: str, YTReplacementDomain: str,
showPublishedDateOnly: bool, showPublishedDateOnly: bool,
newswire: {}, extraJson=None, newswire: {}, dormantMonths: int,
pageNumber=None, maxItemsPerPage=None) -> str: extraJson=None, pageNumber=None,
maxItemsPerPage=None) -> str:
"""Show the profile page as html """Show the profile page as html
""" """
nickname = profileJson['preferredUsername'] nickname = profileJson['preferredUsername']
@ -628,7 +630,8 @@ def htmlProfile(rssIconAtTop: bool,
domain, port, session, domain, port, session,
wfRequest, personCache, extraJson, wfRequest, personCache, extraJson,
projectVersion, ["unfollow"], selected, projectVersion, ["unfollow"], selected,
usersPath, pageNumber, maxItemsPerPage) usersPath, pageNumber, maxItemsPerPage,
dormantMonths)
elif selected == 'followers': elif selected == 'followers':
profileStr += \ profileStr += \
htmlProfileFollowing(translate, baseDir, httpPrefix, htmlProfileFollowing(translate, baseDir, httpPrefix,
@ -637,7 +640,7 @@ def htmlProfile(rssIconAtTop: bool,
wfRequest, personCache, extraJson, wfRequest, personCache, extraJson,
projectVersion, ["block"], projectVersion, ["block"],
selected, usersPath, pageNumber, selected, usersPath, pageNumber,
maxItemsPerPage) maxItemsPerPage, dormantMonths)
elif selected == 'roles': elif selected == 'roles':
profileStr += \ profileStr += \
htmlProfileRoles(translate, nickname, domainFull, htmlProfileRoles(translate, nickname, domainFull,
@ -719,7 +722,8 @@ def htmlProfileFollowing(translate: {}, baseDir: str, httpPrefix: str,
buttons: [], buttons: [],
feedName: str, actor: str, feedName: str, actor: str,
pageNumber: int, pageNumber: int,
maxItemsPerPage: int) -> str: maxItemsPerPage: int,
dormantMonths: int) -> str:
"""Shows following on the profile screen """Shows following on the profile screen
""" """
profileStr = '' profileStr = ''
@ -737,13 +741,22 @@ def htmlProfileFollowing(translate: {}, baseDir: str, httpPrefix: str,
translate['Page up'] + '"></a>\n' + \ translate['Page up'] + '"></a>\n' + \
' </center>\n' ' </center>\n'
for item in followingJson['orderedItems']: for followingActor in followingJson['orderedItems']:
# is this a dormant followed account?
dormant = False
if authorized and feedName == 'following':
dormant = \
isDormant(baseDir, nickname, domain, followingActor,
dormantMonths)
profileStr += \ profileStr += \
individualFollowAsHtml(translate, baseDir, session, individualFollowAsHtml(translate, baseDir, session,
wfRequest, personCache, wfRequest, personCache,
domain, item, authorized, nickname, domain, followingActor,
httpPrefix, projectVersion, authorized, nickname,
httpPrefix, projectVersion, dormant,
buttons) buttons)
if authorized and maxItemsPerPage and pageNumber: if authorized and maxItemsPerPage and pageNumber:
if len(followingJson['orderedItems']) >= maxItemsPerPage: if len(followingJson['orderedItems']) >= maxItemsPerPage:
# page down arrow # page down arrow
@ -1436,12 +1449,15 @@ def individualFollowAsHtml(translate: {},
actorNickname: str, actorNickname: str,
httpPrefix: str, httpPrefix: str,
projectVersion: str, projectVersion: str,
dormant: bool,
buttons=[]) -> str: buttons=[]) -> str:
"""An individual follow entry on the profile screen """An individual follow entry on the profile screen
""" """
nickname = getNicknameFromActor(followUrl) nickname = getNicknameFromActor(followUrl)
domain, port = getDomainFromActor(followUrl) domain, port = getDomainFromActor(followUrl)
titleStr = '@' + nickname + '@' + domain titleStr = '@' + nickname + '@' + domain
if dormant:
titleStr += ' 💤'
avatarUrl = getPersonAvatarUrl(baseDir, followUrl, personCache, True) avatarUrl = getPersonAvatarUrl(baseDir, followUrl, personCache, True)
if not avatarUrl: if not avatarUrl:
avatarUrl = followUrl + '/avatar.png' avatarUrl = followUrl + '/avatar.png'

View File

@ -256,6 +256,7 @@ def htmlSearchSharedItems(cssCache: {}, translate: {},
sharedItemsForm += '</form>\n' sharedItemsForm += '</form>\n'
break break
ctr = 0 ctr = 0
break
if not resultsExist: if not resultsExist:
sharedItemsForm += \ sharedItemsForm += \
'<center><h5>' + translate['No results'] + '</h5></center>\n' '<center><h5>' + translate['No results'] + '</h5></center>\n'
@ -428,6 +429,7 @@ def htmlSkillsSearch(actor: str,
';' + actorJson['icon']['url'] ';' + actorJson['icon']['url']
if indexStr not in results: if indexStr not in results:
results.append(indexStr) results.append(indexStr)
break
if not instanceOnly: if not instanceOnly:
# search actor cache # search actor cache
for subdir, dirs, files in os.walk(baseDir + '/cache/actors/'): for subdir, dirs, files in os.walk(baseDir + '/cache/actors/'):
@ -465,6 +467,7 @@ def htmlSkillsSearch(actor: str,
';' + actorJson['icon']['url'] ';' + actorJson['icon']['url']
if indexStr not in results: if indexStr not in results:
results.append(indexStr) results.append(indexStr)
break
results.sort(reverse=True) results.sort(reverse=True)

View File

@ -429,6 +429,7 @@ def sharesTimelineJson(actor: str, pageNumber: int, itemsPerPage: int,
ctr += 1 ctr += 1
if ctr >= maxSharesPerAccount: if ctr >= maxSharesPerAccount:
break break
break
# sort the shared items in descending order of publication date # sort the shared items in descending order of publication date
sharesJson = OrderedDict(sorted(allSharesJson.items(), reverse=True)) sharesJson = OrderedDict(sorted(allSharesJson.items(), reverse=True))
lastPage = False lastPage = False