indymedia
/
epicyon
mirror of https://gitlab.com/bashrc2/epicyon
7
4
Fork 1
Code Issues 14 Releases Wiki Activity

Option for archiving posts

master
Bob Mottram 2019-07-12 21:43:55 +01:00
parent b89dea3c69
commit 7dfa210023
4 changed files with 80 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
epicyon.py
View File

@ -23,6 +23,7 @@ from posts import archivePosts
from posts import sendPost
from posts import getPublicPostsOfPerson
from posts import getUserUrl
from posts import archivePosts
from session import createSession
from session import getJson
import json
@ -51,6 +52,7 @@ from auth import removePassword
from auth import createPassword
from utils import getDomainFromActor
from utils import getNicknameFromActor
from media import archiveMedia
import argparse
def str2bool(v):
@ -140,6 +142,12 @@ parser.add_argument('--icon','--avatar', dest='avatar', type=str,default=None, \
help='Set the avatar filename for an account')
parser.add_argument('--image','--background', dest='backgroundImage', type=str,default=None, \
help='Set the profile background image for an account')
parser.add_argument('--archive', dest='archive', type=str,default=None, \
help='Archive old files to the given directory')
parser.add_argument('--archiveweeks', dest='archiveWeeks', type=str,default=None, \
help='Specify the number of weeks after which data will be archived')
parser.add_argument('--maxposts', dest='archiveMaxPosts', type=str,default=None, \
help='Maximum number of posts in in/outbox')
args = parser.parse_args()
debug=False
@ -432,6 +440,26 @@ if args.backgroundImage:
print('Background image was not added for '+args.nickname)
sys.exit()
archiveWeeks=4
if args.archiveWeeks:
archiveWeeks=args.archiveWeeks
archiveMaxPosts=256
if args.archiveMaxPosts:
archiveMaxPosts=args.archiveMaxPosts
if args.archive:
if args.archive.lower().endswith('null') or \
args.archive.lower().endswith('delete') or \
args.archive.lower().endswith('none'):
args.archive=None
print('Archiving with deletion of old posts...')
else:
print('Archiving to '+args.archive+'...')
archiveMedia(baseDir,args.archive,archiveWeeks)
archivePosts(baseDir,args.archive,archiveMaxPosts)
print('Archiving complete')
sys.exit()
if federationList:
print('Federating with: '+str(federationList))

8
media.py
View File

@ -105,11 +105,17 @@ def archiveMedia(baseDir: str,archiveDirectory: str,maxWeeks=4) -> None:
weeksSinceEpoch=int((currTime - datetime.datetime(1970,1,1)).days/7)
minWeek=weeksSinceEpoch-maxWeeks
if archiveDirectory:
if not os.path.isdir(archiveDirectory):
os.mkdir(archiveDirectory)
if not os.path.isdir(archiveDirectory+'/media'):
os.mkdir(archiveDirectory+'/media')
for subdir, dirs, files in os.walk(baseDir+'/media'):
for weekDir in dirs:
if int(weekDir)<minWeek:
if archiveDirectory:
move(os.path.join(baseDir+'/media', weekDir),archiveDirectory)
move(os.path.join(baseDir+'/media', weekDir),archiveDirectory+'/media')
else:
# archive to /dev/null
rmtree(os.path.join(baseDir+'/media', weekDir))

57
posts.py
View File

@ -269,18 +269,6 @@ def getPosts(session,outboxUrl: str,maxPosts: int,maxMentions: int, \
break
return personPosts
def createBoxArchive(nickname: str,domain: str,baseDir: str, \
boxname: str) -> str:
"""Creates an archive directory for inbox/outbox posts
"""
handle=nickname.lower()+'@'+domain.lower()
if not os.path.isdir(baseDir+'/accounts/'+handle):
os.mkdir(baseDir+'/accounts/'+handle)
boxArchiveDir=baseDir+'/accounts/'+handle+'/'+boxname+'archive'
if not os.path.isdir(boxArchiveDir):
os.mkdir(boxArchiveDir)
return boxArchiveDir
def deleteAllPosts(baseDir: str,nickname: str, domain: str,boxname: str) -> None:
"""Deletes all posts for a person from inbox or outbox
"""
@ -909,15 +897,50 @@ def createBoxBase(baseDir: str,boxname: str, \
return boxHeader
return boxItems
def archivePosts(nickname: str,domain: str,baseDir: str, \
boxname: str,maxPostsInBox=256) -> None:
def archivePosts(baseDir: str,archiveDir: str,maxPostsInBox=256) -> None:
"""Archives posts for all accounts
"""
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts'):
os.mkdir(archiveDir+'/accounts')
for subdir, dirs, files in os.walk(baseDir+'/accounts'):
for handle in dirs:
if '@' in handle:
nickname=handle.split('@')[0]
domain=handle.split('@')[1]
archiveSubdir=None
if archiveDir:
if not os.path.isdir(archiveDir+'/accounts/'+handle):
os.mkdir(archiveDir+'/accounts/'+handle)
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/inbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/inbox')
if not os.path.isdir(archiveDir+'/accounts/'+handle+'/outbox'):
os.mkdir(archiveDir+'/accounts/'+handle+'/outbox')
archiveSubdir=archiveDir+'/accounts/'+handle+'/inbox'
archivePostsForPerson(nickname,domain,baseDir, \
'inbox',archiveSubdir, \
maxPostsInBox)
if archiveDir:
archiveSubdir=archiveDir+'/accounts/'+handle+'/outbox'
archivePostsForPerson(nickname,domain,baseDir, \
'outbox',archiveSubdir, \
maxPostsInBox)
def archivePostsForPerson(nickname: str,domain: str,baseDir: str, \
boxname: str,archiveDir: str,maxPostsInBox=256) -> None:
"""Retain a maximum number of posts within the given box
Move any others to an archive directory
"""
if boxname!='inbox' and boxname!='outbox':
return
if archiveDir:
if not os.path.isdir(archiveDir):
os.mkdir(archiveDir)
boxDir = createPersonDir(nickname,domain,baseDir,boxname)
archiveDir = createBoxArchive(nickname,domain,baseDir,boxname)
postsInBox=sorted(os.listdir(boxDir), reverse=False)
noOfPosts=len(postsInBox)
if noOfPosts<=maxPostsInBox:
@ -926,9 +949,11 @@ def archivePosts(nickname: str,domain: str,baseDir: str, \
for postFilename in postsInBox:
filePath = os.path.join(boxDir, postFilename)
if os.path.isfile(filePath):
if archiveDir:
archivePath = os.path.join(archiveDir, postFilename)
os.rename(filePath,archivePath)
# TODO: possibly archive any associated media files
else:
os.remove(filePath)
noOfPosts -= 1
if noOfPosts <= maxPostsInBox:
break

4
tests.py
View File

@ -808,8 +808,8 @@ def testCreatePerson():
deleteAllPosts(baseDir,nickname,domain,'outbox')
setPreferredNickname(baseDir,nickname,domain,'badger')
setBio(baseDir,nickname,domain,'Randomly roaming in your backyard')
archivePosts(nickname,domain,baseDir,'inbox',4)
archivePosts(nickname,domain,baseDir,'outbox',4)
archivePostsForPerson(nickname,domain,baseDir,'inbox',None,4)
archivePostsForPerson(nickname,domain,baseDir,'outbox',None,4)
createPublicPost(baseDir,nickname, domain, port,httpPrefix, "G'day world!", False, True, clientToServer,None,None,useBlurhash, None, None, 'Not suitable for Vogons')
os.chdir(currDir)