flake8 format

main
Bob Mottram 2020-04-04 14:44:49 +01:00
parent 6d19fa47a1
commit b63bf2c72d
1 changed files with 304 additions and 225 deletions

185
utils.py
View File

@ -13,16 +13,19 @@ import datetime
import json
from calendar import monthrange
def removeAvatarFromCache(baseDir: str, actorStr: str) -> None:
"""Removes any existing avatar entries from the cache
This avoids duplicate entries with differing extensions
"""
avatarFilenameExtensions = ('png', 'jpg', 'gif', 'webp')
for extension in avatarFilenameExtensions:
avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.'+extension
avatarFilename = \
baseDir + '/cache/avatars/' + actorStr + '.' + extension
if os.path.isfile(avatarFilename):
os.remove(avatarFilename)
def saveJson(jsonObject: {}, filename: str) -> bool:
"""Saves json to a file
"""
@ -32,12 +35,13 @@ def saveJson(jsonObject: {},filename: str) -> bool:
with open(filename, 'w') as fp:
fp.write(json.dumps(jsonObject))
return True
except:
except BaseException:
print('WARN: saveJson ' + str(tries))
time.sleep(1)
tries += 1
return False
def loadJson(filename: str, delaySec=2) -> {}:
"""Makes a few attempts to load a json formatted file
"""
@ -49,14 +53,16 @@ def loadJson(filename: str,delaySec=2) -> {}:
data = fp.read()
jsonObject = json.loads(data)
break
except:
except BaseException:
print('WARN: loadJson exception')
if delaySec > 0:
time.sleep(delaySec)
tries += 1
return jsonObject
def loadJsonOnionify(filename: str,domain: str,onionDomain: str,delaySec=2) -> {}:
def loadJsonOnionify(filename: str, domain: str, onionDomain: str,
delaySec=2) -> {}:
"""Makes a few attempts to load a json formatted file
This also converts the domain name to the onion domain
"""
@ -67,25 +73,33 @@ def loadJsonOnionify(filename: str,domain: str,onionDomain: str,delaySec=2) -> {
with open(filename, 'r') as fp:
data = fp.read()
if data:
data=data.replace(domain,onionDomain).replace('https:','http:')
data = data.replace(domain, onionDomain)
data = data.replace('https:', 'http:')
print('*****data: ' + data)
jsonObject = json.loads(data)
break
except:
except BaseException:
print('WARN: loadJson exception')
if delaySec > 0:
time.sleep(delaySec)
tries += 1
return jsonObject
def getStatusNumber() -> (str, str):
"""Returns the status number and published date
"""
currTime = datetime.datetime.utcnow()
daysSinceEpoch = (currTime - datetime.datetime(1970, 1, 1)).days
# status is the number of seconds since epoch
statusNumber=str(((daysSinceEpoch*24*60*60) + (currTime.hour*60*60) + (currTime.minute*60) + currTime.second)*1000 + int(currTime.microsecond/1000))
# See https://github.com/tootsuite/mastodon/blob/995f8b389a66ab76ec92d9a240de376f1fc13a38/lib/mastodon/snowflake.rb
statusNumber = \
str(((daysSinceEpoch * 24 * 60 * 60) +
(currTime.hour * 60 * 60) +
(currTime.minute * 60) +
currTime.second) * 1000 +
int(currTime.microsecond / 1000))
# See https://github.com/tootsuite/mastodon/blob/
# 995f8b389a66ab76ec92d9a240de376f1fc13a38/lib/mastodon/snowflake.rb
# use the leftover microseconds as the sequence number
sequenceId = currTime.microsecond % 1000
# shift by 16bits "sequence data"
@ -93,8 +107,11 @@ def getStatusNumber() -> (str,str):
published = currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
return statusNumber, published
def evilIncarnate() -> []:
return ('gab.com','gabfed.com','spinster.xyz','kiwifarms.cc','djitter.com')
return ('gab.com', 'gabfed.com', 'spinster.xyz',
'kiwifarms.cc', 'djitter.com')
def isEvil(domain: str) -> bool:
if not isinstance(domain, str):
@ -107,7 +124,9 @@ def isEvil(domain: str) -> bool:
return True
return False
def createPersonDir(nickname: str,domain: str,baseDir: str,dirname: str) -> str:
def createPersonDir(nickname: str, domain: str, baseDir: str,
dirname: str) -> str:
"""Create a directory for a person
"""
handle = nickname + '@' + domain
@ -118,16 +137,19 @@ def createPersonDir(nickname: str,domain: str,baseDir: str,dirname: str) -> str:
os.mkdir(boxDir)
return boxDir
def createOutboxDir(nickname: str, domain: str, baseDir: str) -> str:
"""Create an outbox for a person
"""
return createPersonDir(nickname, domain, baseDir, 'outbox')
def createInboxQueueDir(nickname: str, domain: str, baseDir: str) -> str:
"""Create an inbox queue and returns the feed filename and directory
"""
return createPersonDir(nickname, domain, baseDir, 'queue')
def domainPermitted(domain: str, federationList: []):
if len(federationList) == 0:
return True
@ -137,6 +159,7 @@ def domainPermitted(domain: str, federationList: []):
return True
return False
def urlPermitted(url: str, federationList: [], capability: str):
if isEvil(url):
return False
@ -147,6 +170,7 @@ def urlPermitted(url: str,federationList: [],capability: str):
return True
return False
def getDisplayName(baseDir: str, actor: str, personCache: {}) -> str:
"""Returns the display name for the given actor
"""
@ -159,7 +183,8 @@ def getDisplayName(baseDir: str,actor: str,personCache: {}) -> str:
return personCache[actor]['actor']['name']
else:
# Try to obtain from the cached actors
cachedActorFilename=baseDir+'/cache/actors/'+(actor.replace('/','#'))+'.json'
cachedActorFilename = \
baseDir + '/cache/actors/' + (actor.replace('/', '#')) + '.json'
if os.path.isfile(cachedActorFilename):
actorJson = loadJson(cachedActorFilename, 1)
if actorJson:
@ -167,6 +192,7 @@ def getDisplayName(baseDir: str,actor: str,personCache: {}) -> str:
return(actorJson['name'])
return None
def getNicknameFromActor(actor: str) -> str:
"""Returns the nickname from an actor url
"""
@ -196,26 +222,30 @@ def getNicknameFromActor(actor: str) -> str:
else:
return nickStr.split('/')[0]
def getDomainFromActor(actor: str) -> (str, int):
"""Returns the domain name from an actor url
"""
port = None
if '/profile/' in actor:
domain= \
actor.split('/profile/')[0].replace('https://','').replace('http://','').replace('i2p://','').replace('dat://','')
domain = actor.split('/profile/')[0].replace('https://', '')
domain = domain.replace('http://', '').replace('i2p://', '')
domain = domain.replace('dat://', '')
else:
if '/channel/' in actor:
domain= \
actor.split('/channel/')[0].replace('https://','').replace('http://','').replace('i2p://','').replace('dat://','')
domain = actor.split('/channel/')[0].replace('https://', '')
domain = domain.replace('http://', '').replace('i2p://', '')
domain = domain.replace('dat://', '')
else:
if '/users/' not in actor:
domain= \
actor.replace('https://','').replace('http://','').replace('i2p://','').replace('dat://','')
domain = actor.replace('https://', '').replace('http://', '')
domain = domain.replace('i2p://', '').replace('dat://', '')
if '/' in actor:
domain = domain.split('/')[0]
else:
domain= \
actor.split('/users/')[0].replace('https://','').replace('http://','').replace('i2p://','').replace('dat://','')
domain = actor.split('/users/')[0].replace('https://', '')
domain = domain.replace('http://', '').replace('i2p://', '')
domain = domain.replace('dat://', '')
if ':' in domain:
portStr = domain.split(':')[1]
if not portStr.isdigit():
@ -224,16 +254,18 @@ def getDomainFromActor(actor: str) -> (str,int):
domain = domain.split(':')[0]
return domain, port
def followPerson(baseDir: str,nickname: str, domain: str, \
followNickname: str, followDomain: str, \
federationList: [],debug: bool, \
def followPerson(baseDir: str, nickname: str, domain: str,
followNickname: str, followDomain: str,
federationList: [], debug: bool,
followFile='following.txt') -> bool:
"""Adds a person to the follow list
"""
if not domainPermitted(followDomain.lower().replace('\n',''), \
if not domainPermitted(followDomain.lower().replace('\n', ''),
federationList):
if debug:
print('DEBUG: follow of domain '+followDomain+' not permitted')
print('DEBUG: follow of domain ' +
followDomain + ' not permitted')
return False
if debug:
print('DEBUG: follow of domain ' + followDomain)
@ -285,14 +317,17 @@ def followPerson(baseDir: str,nickname: str, domain: str, \
print('DEBUG: follow added')
return True
except Exception as e:
print('WARN: Failed to write entry to follow file '+filename+' '+str(e))
print('WARN: Failed to write entry to follow file ' +
filename + ' ' + str(e))
if debug:
print('DEBUG: creating new following file to follow ' + handleToFollow)
with open(filename, "w") as followfile:
followfile.write(handleToFollow + '\n')
return True
def locatePost(baseDir: str,nickname: str,domain: str,postUrl: str,replies=False) -> str:
def locatePost(baseDir: str, nickname: str, domain: str,
postUrl: str, replies=False) -> str:
"""Returns the filename for the given status post url
"""
if not replies:
@ -301,43 +336,49 @@ def locatePost(baseDir: str,nickname: str,domain: str,postUrl: str,replies=False
extension = 'replies'
# if this post in the shared inbox?
handle='inbox@'+domain
postUrl = postUrl.replace('/', '#').replace('/activity', '').strip()
boxName = 'inbox'
postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
postFilename = baseDir + '/accounts/' + nickname + '@' + domain + \
'/' + boxName + '/' + postUrl + '.' + extension
if os.path.isfile(postFilename):
return postFilename
boxName = 'outbox'
postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
postFilename = baseDir + '/accounts/' + nickname + '@' + domain + \
'/' + boxName + '/' + postUrl + '.' + extension
if os.path.isfile(postFilename):
return postFilename
boxName = 'tlblogs'
postFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/'+boxName+'/'+postUrl+'.'+extension
postFilename = baseDir + '/accounts/' + nickname + '@' + domain + \
'/' + boxName + '/' + postUrl + '.'+extension
if os.path.isfile(postFilename):
return postFilename
postFilename=baseDir+'/cache/announce/'+nickname+'/'+postUrl+'.'+extension
postFilename = baseDir + '/cache/announce/' + \
nickname + '/' + postUrl + '.' + extension
if os.path.isfile(postFilename):
return postFilename
print('WARN: unable to locate '+nickname+' '+postUrl+'.'+extension)
print('WARN: unable to locate ' + nickname + ' ' +
postUrl + '.' + extension)
return None
def removeAttachment(baseDir: str, httpPrefix: str, domain: str, postJson: {}):
if not postJson.get('attachment'):
return
if not postJson['attachment'][0].get('url'):
return
if port:
if port!=80 and port!=443:
if ':' not in domain:
domain=domain+':'+str(port)
# if port:
# if port != 80 and port != 443:
# if ':' not in domain:
# domain = domain + ':' + str(port)
attachmentUrl = postJson['attachment'][0]['url']
if not attachmentUrl:
return
mediaFilename=baseDir+'/'+attachmentUrl.replace(httpPrefix+'://'+domain+'/','')
mediaFilename = baseDir + '/' + \
attachmentUrl.replace(httpPrefix + '://' + domain + '/', '')
if os.path.isfile(mediaFilename):
os.remove(mediaFilename)
etagFilename = mediaFilename + '.etag'
@ -345,7 +386,9 @@ def removeAttachment(baseDir: str,httpPrefix: str,domain: str,postJson: {}):
os.remove(etagFilename)
postJson['attachment'] = []
def removeModerationPostFromIndex(baseDir: str,postUrl: str,debug: bool) -> None:
def removeModerationPostFromIndex(baseDir: str, postUrl: str,
debug: bool) -> None:
"""Removes a url from the moderation index
"""
moderationIndexFile = baseDir + '/accounts/moderation.txt'
@ -361,15 +404,21 @@ def removeModerationPostFromIndex(baseDir: str,postUrl: str,debug: bool) -> None
f.write(line)
else:
if debug:
print('DEBUG: removed '+postId+' from moderation index')
print('DEBUG: removed ' + postId +
' from moderation index')
def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilename: str,debug: bool) -> None:
def deletePost(baseDir: str, httpPrefix: str,
nickname: str, domain: str, postFilename: str,
debug: bool) -> None:
"""Recursively deletes a post and its replies and attachments
"""
postJsonObject = loadJson(postFilename, 1)
if postJsonObject:
# don't allow deletion of bookmarked posts
bookmarksIndexFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/bookmarks.index'
bookmarksIndexFilename = \
baseDir + '/accounts/' + nickname + '@' + domain + \
'/bookmarks.index'
if os.path.isfile(bookmarksIndexFilename):
bookmarkIndex = postFilename.split('/')[-1] + '\n'
if bookmarkIndex in open(bookmarksIndexFilename).read():
@ -411,16 +460,19 @@ def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilena
if '#' in postJsonObject['object']['content']:
removeHashtagIndex = True
if removeHashtagIndex:
if postJsonObject['object'].get('id') and postJsonObject['object'].get('tag'):
if postJsonObject['object'].get('id') and \
postJsonObject['object'].get('tag'):
# get the id of the post
postId=postJsonObject['object']['id'].replace('/activity','')
postId = \
postJsonObject['object']['id'].replace('/activity', '')
for tag in postJsonObject['object']['tag']:
if tag['type'] != 'Hashtag':
continue
if not tag.get('name'):
continue
# find the index file for this tag
tagIndexFilename=baseDir+'/tags/'+tag['name'][1:]+'.txt'
tagIndexFilename = \
baseDir + '/tags/' + tag['name'][1:] + '.txt'
if not os.path.isfile(tagIndexFilename):
continue
# remove postId from the tag index file
@ -434,7 +486,8 @@ def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilena
continue
newlines += l
if not newlines.strip():
# if there are no lines then remove the hashtag file
# if there are no lines then remove the
# hashtag file
os.remove(tagIndexFilename)
else:
with open(tagIndexFilename, "w+") as f:
@ -450,24 +503,35 @@ def deletePost(baseDir: str,httpPrefix: str,nickname: str,domain: str,postFilena
replyFile = locatePost(baseDir, nickname, domain, replyId)
if replyFile:
if os.path.isfile(replyFile):
deletePost(baseDir,httpPrefix,nickname,domain,replyFile,debug)
deletePost(baseDir, httpPrefix,
nickname, domain, replyFile, debug)
# remove the replies file
os.remove(repliesFilename)
# finally, remove the post itself
os.remove(postFilename)
def validNickname(domain: str, nickname: str) -> bool:
forbiddenChars=['.',' ','/','?',':',';','@']
forbiddenChars = ('.', ' ', '/', '?', ':', ';', '@')
for c in forbiddenChars:
if c in nickname:
return False
if nickname == domain:
return False
reservedNames=['inbox','dm','outbox','following','public','followers','profile','channel','capabilities','calendar','tlreplies','tlmedia','tlblogs','moderation','activity','undo','reply','replies','question','like','likes','users','statuses','updates','repeat','announce','shares']
reservedNames = ('inbox', 'dm', 'outbox', 'following',
'public', 'followers', 'profile',
'channel', 'capabilities', 'calendar',
'tlreplies', 'tlmedia', 'tlblogs',
'moderation', 'activity', 'undo',
'reply', 'replies', 'question', 'like',
'likes', 'users', 'statuses',
'updates', 'repeat', 'announce',
'shares')
if nickname in reservedNames:
return False
return True
def noOfAccounts(baseDir: str) -> bool:
"""Returns the number of accounts on the system
"""
@ -479,6 +543,7 @@ def noOfAccounts(baseDir: str) -> bool:
accountCtr += 1
return accountCtr
def noOfActiveAccountsMonthly(baseDir: str, months: int) -> bool:
"""Returns the number of accounts on the system this month
"""
@ -489,7 +554,8 @@ def noOfActiveAccountsMonthly(baseDir: str,months: int) -> bool:
for account in dirs:
if '@' in account:
if not account.startswith('inbox@'):
lastUsedFilename=baseDir+'/accounts/'+account+'/.lastUsed'
lastUsedFilename = \
baseDir + '/accounts/' + account + '/.lastUsed'
if os.path.isfile(lastUsedFilename):
with open(lastUsedFilename, 'r') as lastUsedFile:
lastUsed = lastUsedFile.read()
@ -499,7 +565,9 @@ def noOfActiveAccountsMonthly(baseDir: str,months: int) -> bool:
accountCtr += 1
return accountCtr
def isPublicPostFromUrl(baseDir: str,nickname: str,domain: str,postUrl: str) -> bool:
def isPublicPostFromUrl(baseDir: str, nickname: str, domain: str,
postUrl: str) -> bool:
"""Returns whether the given url is a public post
"""
postFilename = locatePost(baseDir, nickname, domain, postUrl)
@ -510,6 +578,7 @@ def isPublicPostFromUrl(baseDir: str,nickname: str,domain: str,postUrl: str) ->
return False
return isPublicPost(postJsonObject)
def isPublicPost(postJsonObject: {}) -> bool:
"""Returns true if the given post is public
"""
@ -528,6 +597,7 @@ def isPublicPost(postJsonObject: {}) -> bool:
return True
return False
def copytree(src: str, dst: str, symlinks=False, ignore=None):
"""Copy a directory
"""
@ -539,13 +609,16 @@ def copytree(src: str, dst: str, symlinks=False, ignore=None):
else:
shutil.copy2(s, d)
def getCachedPostDirectory(baseDir: str, nickname: str, domain: str) -> str:
"""Returns the directory where the html post cache exists
"""
htmlPostCacheDir=baseDir+'/accounts/'+nickname+'@'+domain+'/postcache'
htmlPostCacheDir = baseDir + '/accounts/' + \
nickname + '@' + domain + '/postcache'
return htmlPostCacheDir
def getCachedPostFilename(baseDir: str,nickname: str,domain: str, \
def getCachedPostFilename(baseDir: str, nickname: str, domain: str,
postJsonObject: {}) -> str:
"""Returns the html cache filename for the given post
"""
@ -562,6 +635,7 @@ def getCachedPostFilename(baseDir: str,nickname: str,domain: str, \
cachedPostFilename = cachedPostFilename + '.html'
return cachedPostFilename
def removePostFromCache(postJsonObject: {}, recentPostsCache: {}):
""" if the post exists in the recent posts cache then remove it
"""
@ -584,7 +658,8 @@ def removePostFromCache(postJsonObject: {},recentPostsCache: {}):
del recentPostsCache['html'][postId]
recentPostsCache['index'].remove(postId)
def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
def updateRecentPostsCache(recentPostsCache: {}, maxRecentPosts: int,
postJsonObject: {}, htmlStr: str) -> None:
"""Store recent posts in memory so that they can be quickly recalled
"""
@ -613,6 +688,7 @@ def updateRecentPostsCache(recentPostsCache: {},maxRecentPosts: int, \
recentPostsCache['json'][postId] = json.dumps(postJsonObject)
recentPostsCache['html'][postId] = htmlStr
def fileLastModified(filename: str) -> str:
"""Returns the date when a file was last modified
"""
@ -620,6 +696,7 @@ def fileLastModified(filename: str) -> str:
modifiedTime = datetime.datetime.fromtimestamp(t)
return modifiedTime.strftime("%Y-%m-%dT%H:%M:%SZ")
def daysInMonth(year: int, monthNumber: int) -> int:
"""Returns the number of days in the month
"""
@ -628,12 +705,14 @@ def daysInMonth(year: int,monthNumber: int) -> int:
daysRange = monthrange(year, monthNumber)
return daysRange[1]
def mergeDicts(dict1: {}, dict2: {}) -> {}:
"""Merges two dictionaries
"""
res = {**dict1, **dict2}
return res
def isBlogPost(postJsonObject: {}) -> bool:
"""Is the given post a blog post?
"""