diff --git a/daemon.py b/daemon.py index e9628538c..8a886a86f 100644 --- a/daemon.py +++ b/daemon.py @@ -3172,7 +3172,8 @@ class PubServer(BaseHTTPRequestHandler): httpPrefix, domainFull, actorStr, callingDomain, - sharedItemsFederatedDomains) + sharedItemsFederatedDomains, + 'shares') if sharedItemsStr: msg = sharedItemsStr.encode('utf-8') msglen = len(msg) @@ -3390,7 +3391,7 @@ class PubServer(BaseHTTPRequestHandler): shareDomain, sharePort = getDomainFromActor(shareActor) removeSharedItem(baseDir, shareNickname, shareDomain, itemID, - httpPrefix, domainFull) + httpPrefix, domainFull, 'shares') if callingDomain.endswith('.onion') and onionDomain: originPathStr = 'http://' + onionDomain + usersPath @@ -9488,12 +9489,12 @@ class PubServer(BaseHTTPRequestHandler): onionDomain: str, i2pDomain: str, GETstartTime, GETtimings: {}, proxyType: str, cookie: str, - debug: str) -> bool: + debug: str, sharesFileType: str) -> bool: """Shows the shares feed """ shares = \ getSharesFeedForPerson(baseDir, domain, port, path, - httpPrefix, sharesPerPage) + httpPrefix, sharesFileType, sharesPerPage) if shares: if self._requestHTTP(): pageNumber = 1 @@ -9503,7 +9504,7 @@ class PubServer(BaseHTTPRequestHandler): shares = \ getSharesFeedForPerson(baseDir, domain, port, path + '?page=true', - httpPrefix, + httpPrefix, sharesFileType, sharesPerPage) else: pageNumberStr = path.split('?page=')[1] @@ -9514,7 +9515,7 @@ class PubServer(BaseHTTPRequestHandler): searchPath = path.split('?page=')[0] getPerson = \ personLookup(domain, - searchPath.replace('/shares', ''), + searchPath.replace('/' + sharesFileType, ''), baseDir) if getPerson: if not self.server.session: @@ -9549,7 +9550,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.projectVersion, baseDir, httpPrefix, authorized, - getPerson, 'shares', + getPerson, sharesFileType, self.server.session, self.server.cachedWebfingers, self.server.personCache, @@ -10865,7 +10866,7 @@ class PubServer(BaseHTTPRequestHandler): sharesCatalogEndpoint(self.server.baseDir, self.server.httpPrefix, self.server.domainFull, - self.path) + self.path, 'shares') else: domainFull = self.server.domainFull httpPrefix = self.server.httpPrefix @@ -10881,7 +10882,8 @@ class PubServer(BaseHTTPRequestHandler): self.server.domain, domainFull, self.path, - self.server.debug) + self.server.debug, + 'shares') msg = json.dumps(catalogJson, ensure_ascii=False).encode('utf-8') msglen = len(msg) @@ -10895,7 +10897,8 @@ class PubServer(BaseHTTPRequestHandler): sharesCatalogCSVEndpoint(self.server.baseDir, self.server.httpPrefix, self.server.domainFull, - self.path).encode('utf-8') + self.path, + 'shares').encode('utf-8') msglen = len(msg) self._set_headers('text/csv', msglen, None, callingDomain) @@ -11363,7 +11366,7 @@ class PubServer(BaseHTTPRequestHandler): itemID, self.server.translate, self.server.sharedItemsFederatedDomains, self.server.defaultTimeline, - self.server.themeName) + self.server.themeName, 'shares') if not msg: if callingDomain.endswith('.onion') and \ self.server.onionDomain: @@ -11396,7 +11399,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.translate, self.server.baseDir, actor, itemID, - callingDomain) + callingDomain, 'shares') if not msg: if callingDomain.endswith('.onion') and \ self.server.onionDomain: @@ -13145,7 +13148,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.i2pDomain, GETstartTime, GETtimings, self.server.proxyType, - cookie, self.server.debug): + cookie, self.server.debug, 'shares'): return self._benchmarkGETtimings(GETstartTime, GETtimings, @@ -14009,7 +14012,7 @@ class PubServer(BaseHTTPRequestHandler): self.server.debug, city, itemPrice, itemCurrency, self.server.systemLanguage, - self.server.translate) + self.server.translate, 'shares') if filename: if os.path.isfile(filename): os.remove(filename) diff --git a/epicyon.py b/epicyon.py index 38adf5675..da19a246d 100644 --- a/epicyon.py +++ b/epicyon.py @@ -2381,7 +2381,7 @@ if args.testdata: "mechanical", "City", "0", "GBP", "2 months", - debug, city, args.language, {}) + debug, city, args.language, {}, 'shares') addShare(baseDir, httpPrefix, nickname, domain, port, "witch hat", @@ -2391,7 +2391,7 @@ if args.testdata: "clothing", "City", "0", "GBP", "3 months", - debug, city, args.language, {}) + debug, city, args.language, {}, 'shares') deleteAllPosts(baseDir, nickname, domain, 'inbox') deleteAllPosts(baseDir, nickname, domain, 'outbox') diff --git a/shares.py b/shares.py index 321f4d852..779f81dbd 100644 --- a/shares.py +++ b/shares.py @@ -36,6 +36,7 @@ from utils import isAccountDir from utils import acctDir from utils import isfloat from utils import getCategoryTypes +from utils import getSharesFilesList from media import processMetaData from filters import isFilteredGlobally from siteactive import siteIsActive @@ -113,17 +114,22 @@ def _getValidSharedItemID(actor: str, displayName: str) -> str: def removeSharedItem(baseDir: str, nickname: str, domain: str, itemID: str, - httpPrefix: str, domainFull: str) -> None: + httpPrefix: str, domainFull: str, + sharesFileType: str) -> None: """Removes a share for a person """ - sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' + sharesFilename = \ + acctDir(baseDir, nickname, domain) + '/' + sharesFileType + '.json' if not os.path.isfile(sharesFilename): - print('ERROR: missing shares.json ' + sharesFilename) + print('ERROR: remove shared item, missing ' + + sharesFileType + '.json ' + sharesFilename) return sharesJson = loadJson(sharesFilename) if not sharesJson: - print('ERROR: shares.json could not be loaded from ' + sharesFilename) + print('ERROR: remove shared item, ' + + sharesFileType + '.json could not be loaded from ' + + sharesFilename) return if sharesJson.get(itemID): @@ -279,7 +285,8 @@ def addShare(baseDir: str, itemQty: float, itemType: str, itemCategory: str, location: str, duration: str, debug: bool, city: str, price: str, currency: str, - systemLanguage: str, translate: {}) -> None: + systemLanguage: str, translate: {}, + sharesFileType: str) -> None: """Adds a new share """ if isFilteredGlobally(baseDir, @@ -287,7 +294,8 @@ def addShare(baseDir: str, itemType + ' ' + itemCategory): print('Shared item was filtered due to content') return - sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' + sharesFilename = \ + acctDir(baseDir, nickname, domain) + '/' + sharesFileType + '.json' sharesJson = {} if os.path.isfile(sharesFilename): sharesJson = loadJson(sharesFilename, 1, 2) @@ -367,16 +375,20 @@ def expireShares(baseDir: str) -> None: continue nickname = account.split('@')[0] domain = account.split('@')[1] - _expireSharesForAccount(baseDir, nickname, domain) + for sharesFileType in getSharesFilesList(): + _expireSharesForAccount(baseDir, nickname, domain, + sharesFileType) break -def _expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None: +def _expireSharesForAccount(baseDir: str, nickname: str, domain: str, + sharesFileType: str) -> None: """Removes expired items from shares for a particular account """ handleDomain = removeDomainPort(domain) handle = nickname + '@' + handleDomain - sharesFilename = baseDir + '/accounts/' + handle + '/shares.json' + sharesFilename = \ + baseDir + '/accounts/' + handle + '/' + sharesFileType + '.json' if not os.path.isfile(sharesFilename): return sharesJson = loadJson(sharesFilename, 1, 2) @@ -403,10 +415,11 @@ def _expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None: def getSharesFeedForPerson(baseDir: str, domain: str, port: int, path: str, httpPrefix: str, - sharesPerPage=12) -> {}: + sharesFileType: str, + sharesPerPage: int = 12) -> {}: """Returns the shares for an account from GET requests """ - if '/shares' not in path: + if '/' + sharesFileType not in path: return None # handle page numbers headerOnly = True @@ -423,13 +436,15 @@ def getSharesFeedForPerson(baseDir: str, path = path.split('?page=')[0] headerOnly = False - if not path.endswith('/shares'): + if not path.endswith('/' + sharesFileType): return None nickname = None if path.startswith('/users/'): - nickname = path.replace('/users/', '', 1).replace('/shares', '') + nickname = \ + path.replace('/users/', '', 1).replace('/' + sharesFileType, '') if path.startswith('/@'): - nickname = path.replace('/@', '', 1).replace('/shares', '') + nickname = \ + path.replace('/@', '', 1).replace('/' + sharesFileType, '') if not nickname: return None if not validNickname(domain, nickname): @@ -438,7 +453,9 @@ def getSharesFeedForPerson(baseDir: str, domain = getFullDomain(domain, port) handleDomain = removeDomainPort(domain) - sharesFilename = acctDir(baseDir, nickname, handleDomain) + '/shares.json' + sharesFilename = \ + acctDir(baseDir, nickname, handleDomain) + '/' + \ + sharesFileType + '.json' if headerOnly: noOfShares = 0 @@ -449,8 +466,8 @@ def getSharesFeedForPerson(baseDir: str, idStr = httpPrefix + '://' + domain + '/users/' + nickname shares = { '@context': 'https://www.w3.org/ns/activitystreams', - 'first': idStr + '/shares?page=1', - 'id': idStr + '/shares', + 'first': idStr + '/' + sharesFileType + '?page=1', + 'id': idStr + '/' + sharesFileType, 'totalItems': str(noOfShares), 'type': 'OrderedCollection' } @@ -463,15 +480,14 @@ def getSharesFeedForPerson(baseDir: str, idStr = httpPrefix + '://' + domain + '/users/' + nickname shares = { '@context': 'https://www.w3.org/ns/activitystreams', - 'id': idStr + '/shares?page=' + str(pageNumber), + 'id': idStr + '/' + sharesFileType + '?page=' + str(pageNumber), 'orderedItems': [], - 'partOf': idStr + '/shares', + 'partOf': idStr + '/' + sharesFileType, 'totalItems': 0, 'type': 'OrderedCollectionPage' } if not os.path.isfile(sharesFilename): - print("test5") return shares currPage = 1 pageCtr = 0 @@ -495,7 +511,7 @@ def getSharesFeedForPerson(baseDir: str, if nextPageNumber > lastPage: shares['next'] = \ httpPrefix + '://' + domain + '/users/' + nickname + \ - '/shares?page=' + str(lastPage) + '/' + sharesFileType + '?page=' + str(lastPage) return shares @@ -816,7 +832,7 @@ def outboxShareUpload(baseDir: str, httpPrefix: str, debug, city, messageJson['object']['itemPrice'], messageJson['object']['itemCurrency'], - systemLanguage, translate) + systemLanguage, translate, 'shares') if debug: print('DEBUG: shared item received via c2s') @@ -847,7 +863,7 @@ def outboxUndoShareUpload(baseDir: str, httpPrefix: str, domainFull = getFullDomain(domain, port) removeSharedItem(baseDir, nickname, domain, messageJson['object']['displayName'], - httpPrefix, domainFull) + httpPrefix, domainFull, 'shares') if debug: print('DEBUG: shared item removed via c2s') @@ -886,7 +902,8 @@ def _sharesCatalogParams(path: str) -> (bool, float, float, str): def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str, nickname: str, domain: str, domainFull: str, - path: str, debug: bool) -> {}: + path: str, debug: bool, + sharesFileType: str) -> {}: """Returns the endpoint for the shares catalog of a particular account See https://github.com/datafoodconsortium/ontology """ @@ -896,7 +913,10 @@ def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str, dfcPtUrl = \ "http://static.datafoodconsortium.org/data/productTypes.rdf#" owner = httpPrefix + '://' + domainFull + '/users/' + nickname - dfcInstanceId = owner + '/catalog' + if sharesFileType == 'shares': + dfcInstanceId = owner + '/catalog' + else: + dfcInstanceId = owner + '/wantedItems' endpoint = { "@context": { "DFC": dfcUrl, @@ -911,10 +931,11 @@ def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str, currDate = datetime.datetime.utcnow() currDateStr = currDate.strftime("%Y-%m-%d") - sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' + sharesFilename = \ + acctDir(baseDir, nickname, domain) + '/' + sharesFileType + '.json' if not os.path.isfile(sharesFilename): if debug: - print('shares.json file not found: ' + sharesFilename) + print(sharesFileType + '.json file not found: ' + sharesFilename) return endpoint sharesJson = loadJson(sharesFilename, 1, 2) if not sharesJson: @@ -970,7 +991,7 @@ def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str, def sharesCatalogEndpoint(baseDir: str, httpPrefix: str, domainFull: str, - path: str) -> {}: + path: str, sharesFileType: str) -> {}: """Returns the endpoint for the shares catalog for the instance See https://github.com/datafoodconsortium/ontology """ @@ -1003,7 +1024,8 @@ def sharesCatalogEndpoint(baseDir: str, httpPrefix: str, owner = httpPrefix + '://' + domainFull + '/users/' + nickname sharesFilename = \ - acctDir(baseDir, nickname, domain) + '/shares.json' + acctDir(baseDir, nickname, domain) + '/' + \ + sharesFileType + '.json' if not os.path.isfile(sharesFilename): continue print('Test 78363 ' + sharesFilename) @@ -1056,11 +1078,12 @@ def sharesCatalogEndpoint(baseDir: str, httpPrefix: str, def sharesCatalogCSVEndpoint(baseDir: str, httpPrefix: str, domainFull: str, - path: str) -> str: + path: str, sharesFileType: str) -> str: """Returns a CSV version of the shares catalog """ catalogJson = \ - sharesCatalogEndpoint(baseDir, httpPrefix, domainFull, path) + sharesCatalogEndpoint(baseDir, httpPrefix, domainFull, path, + sharesFileType) if not catalogJson: return '' if not catalogJson.get('DFC:supplies'): @@ -1257,7 +1280,8 @@ def _updateFederatedSharesCache(session, sharedItemsFederatedDomains: [], baseDir: str, domainFull: str, httpPrefix: str, tokensJson: {}, debug: bool, - systemLanguage: str) -> None: + systemLanguage: str, + sharesFileType: str) -> None: """Updates the cache of federated shares for the instance. This enables shared items to be available even when other instances might not be online @@ -1266,7 +1290,10 @@ def _updateFederatedSharesCache(session, sharedItemsFederatedDomains: [], cacheDir = baseDir + '/cache' if not os.path.isdir(cacheDir): os.mkdir(cacheDir) - catalogsDir = cacheDir + '/catalogs' + if sharesFileType == 'shares': + catalogsDir = cacheDir + '/catalogs' + else: + catalogsDir = cacheDir + '/wantedItems' if not os.path.isdir(catalogsDir): os.mkdir(catalogsDir) @@ -1285,7 +1312,10 @@ def _updateFederatedSharesCache(session, sharedItemsFederatedDomains: [], continue if not siteIsActive(httpPrefix + '://' + federatedDomainFull): continue - url = httpPrefix + '://' + federatedDomainFull + '/catalog' + if sharesFileType == 'shares': + url = httpPrefix + '://' + federatedDomainFull + '/catalog' + else: + url = httpPrefix + '://' + federatedDomainFull + '/wantedItems' asHeader['Authorization'] = tokensJson[federatedDomainFull] catalogJson = getJson(session, url, asHeader, None, debug, __version__, httpPrefix, None) @@ -1300,7 +1330,8 @@ def _updateFederatedSharesCache(session, sharedItemsFederatedDomains: [], baseDir, systemLanguage) if sharesJson: sharesFilename = \ - catalogsDir + '/' + federatedDomainFull + '.shares.json' + catalogsDir + '/' + federatedDomainFull + '.' + \ + sharesFileType + '.json' saveJson(sharesJson, sharesFilename) print('Converted shares catalog for ' + federatedDomainFull) else: @@ -1441,9 +1472,11 @@ def runFederatedSharesDaemon(baseDir: str, httpd, httpPrefix: str, continue session = createSession(proxyType) - _updateFederatedSharesCache(session, sharedItemsFederatedDomains, - baseDir, domainFull, httpPrefix, - tokensJson, debug, systemLanguage) + for sharesFileType in getSharesFilesList(): + _updateFederatedSharesCache(session, sharedItemsFederatedDomains, + baseDir, domainFull, httpPrefix, + tokensJson, debug, systemLanguage, + sharesFileType) time.sleep(secondsPerHour * 6) diff --git a/utils.py b/utils.py index 0ef95292a..1c0892713 100644 --- a/utils.py +++ b/utils.py @@ -2830,3 +2830,9 @@ def getCategoryTypes(baseDir: str) -> []: categories.append(ontologyFilename.replace('Types', '')) break return categories + + +def getSharesFilesList() -> []: + """Returns the possible shares files + """ + return ('shares', 'wanted') diff --git a/webapp_column_left.py b/webapp_column_left.py index 48a8e9635..3c78dd4db 100644 --- a/webapp_column_left.py +++ b/webapp_column_left.py @@ -43,7 +43,7 @@ def _getLeftColumnShares(baseDir: str, sharesJson, lastPage = \ sharesTimelineJson(actor, pageNumber, maxSharesInLeftColumn, baseDir, domain, nickname, maxSharesInLeftColumn, - sharedItemsFederatedDomains) + sharedItemsFederatedDomains, 'shares') if not sharesJson: return [] diff --git a/webapp_confirm.py b/webapp_confirm.py index c4be6679c..982c8f921 100644 --- a/webapp_confirm.py +++ b/webapp_confirm.py @@ -105,19 +105,21 @@ def htmlConfirmDelete(cssCache: {}, def htmlConfirmRemoveSharedItem(cssCache: {}, translate: {}, baseDir: str, actor: str, itemID: str, - callingDomain: str) -> str: + callingDomain: str, + sharesFileType: str) -> str: """Shows a screen asking to confirm the removal of a shared item """ nickname = getNicknameFromActor(actor) domain, port = getDomainFromActor(actor) domainFull = getFullDomain(domain, port) - sharesFile = acctDir(baseDir, nickname, domain) + '/shares.json' + sharesFile = \ + acctDir(baseDir, nickname, domain) + '/' + sharesFileType + '.json' if not os.path.isfile(sharesFile): - print('ERROR: no shares file ' + sharesFile) + print('ERROR: no ' + sharesFileType + ' file ' + sharesFile) return None sharesJson = loadJson(sharesFile) if not sharesJson: - print('ERROR: unable to load shares.json') + print('ERROR: unable to load ' + sharesFileType + '.json') return None if not sharesJson.get(itemID): print('ERROR: share named "' + itemID + '" is not in ' + sharesFile) @@ -148,7 +150,12 @@ def htmlConfirmRemoveSharedItem(cssCache: {}, translate: {}, baseDir: str, '

' + translate['Remove'] + \ ' ' + sharedItemDisplayName + ' ?

\n' postActor = getAltPath(actor, domainFull, callingDomain) - sharesStr += '
\n' + if sharesFileType == 'shares': + endpoint = 'rmshare' + else: + endpoint = 'rmwanted' + sharesStr += \ + ' \n' sharesStr += \ ' \n' sharesStr += '

' + \ - '' + \ - translate['Shared Items Search'] + \ - '

' + '' + titleStr + '' resultsExist = False for subdir, dirs, files in os.walk(baseDir + '/accounts'): for handle in dirs: @@ -239,7 +242,7 @@ def htmlSearchSharedItems(cssCache: {}, translate: {}, continue contactNickname = handle.split('@')[0] sharesFilename = baseDir + '/accounts/' + handle + \ - '/shares.json' + '/' + sharesFileType + '.json' if not os.path.isfile(sharesFilename): continue @@ -264,13 +267,16 @@ def htmlSearchSharedItems(cssCache: {}, translate: {}, break # search federated shared items - catalogsDir = baseDir + '/cache/catalogs' + if sharesFileType == 'shares': + catalogsDir = baseDir + '/cache/catalogs' + else: + catalogsDir = baseDir + '/cache/wantedItems' if currPage <= pageNumber and os.path.isdir(catalogsDir): for subdir, dirs, files in os.walk(catalogsDir): for f in files: if '#' in f: continue - if not f.endswith('.shares.json'): + if not f.endswith('.' + sharesFileType + '.json'): continue federatedDomain = f.split('.')[0] if federatedDomain not in sharedItemsFederatedDomains: diff --git a/webapp_timeline.py b/webapp_timeline.py index d6c8790e7..1a38f35f6 100644 --- a/webapp_timeline.py +++ b/webapp_timeline.py @@ -699,7 +699,7 @@ def htmlTimeline(cssCache: {}, defaultTimeline: str, _htmlSharesTimeline(translate, pageNumber, itemsPerPage, baseDir, actor, nickname, domain, port, maxSharesPerAccount, httpPrefix, - sharedItemsFederatedDomains) + + sharedItemsFederatedDomains, 'shares') + _htmlTimelineEnd(baseDir, nickname, domainFull, httpPrefix, translate, moderator, editor, @@ -916,13 +916,14 @@ def _htmlSharesTimeline(translate: {}, pageNumber: int, itemsPerPage: int, baseDir: str, actor: str, nickname: str, domain: str, port: int, maxSharesPerAccount: int, httpPrefix: str, - sharedItemsFederatedDomains: []) -> str: + sharedItemsFederatedDomains: [], + sharesFileType: str) -> str: """Show shared items timeline as html """ sharesJson, lastPage = \ sharesTimelineJson(actor, pageNumber, itemsPerPage, baseDir, domain, nickname, maxSharesPerAccount, - sharedItemsFederatedDomains) + sharedItemsFederatedDomains, sharesFileType) domainFull = getFullDomain(domain, port) actor = httpPrefix + '://' + domainFull + '/users/' + nickname adminNickname = getConfigParam(baseDir, 'admin') @@ -935,7 +936,7 @@ def _htmlSharesTimeline(translate: {}, pageNumber: int, itemsPerPage: int, if pageNumber > 1: timelineStr += \ '
\n' + \ - ' str: + defaultTimeline: str, theme: str, + sharesFileType: str) -> str: """Shows an individual shared item after selecting it from the left column """ sharesJson = None @@ -1355,20 +1360,24 @@ def htmlShowShare(baseDir: str, domain: str, nickname: str, if '://' + domainFull + '/' in shareUrl: # shared item on this instance sharesFilename = \ - acctDir(baseDir, contactNickname, domain) + '/shares.json' + acctDir(baseDir, contactNickname, domain) + '/' + \ + sharesFileType + '.json' if not os.path.isfile(sharesFilename): return None sharesJson = loadJson(sharesFilename) else: # federated shared item - catalogsDir = baseDir + '/cache/catalogs' + if sharesFileType == 'shares': + catalogsDir = baseDir + '/cache/catalogs' + else: + catalogsDir = baseDir + '/cache/wantedItems' if not os.path.isdir(catalogsDir): return None for subdir, dirs, files in os.walk(catalogsDir): for f in files: if '#' in f: continue - if not f.endswith('.shares.json'): + if not f.endswith('.' + sharesFileType + '.json'): continue federatedDomain = f.split('.')[0] if federatedDomain not in sharedItemsFederatedDomains: