__filename__ = "shares.py" __author__ = "Bob Mottram" __license__ = "AGPL3+" __version__ = "1.2.0" __maintainer__ = "Bob Mottram" __email__ = "bob@freedombone.net" __status__ = "Production" __module_group__ = "Timeline" import os import time import datetime from webfinger import webfingerHandle from auth import createBasicAuthHeader from posts import getPersonBox from session import postJson from session import postImage from utils import getFullDomain from utils import validNickname from utils import loadJson from utils import saveJson from utils import getImageExtensions from utils import hasObjectDict from utils import removeDomainPort from utils import isAccountDir from utils import acctDir from media import processMetaData def _loadDfcIds(baseDir: str, systemLanguage: str) -> {}: """Loads the product types ontology This is used to add an id to shared items """ productTypesFilename = baseDir + '/ontology/customProductTypes.json' if not os.path.isfile(productTypesFilename): productTypesFilename = baseDir + '/ontology/productTypes.json' productTypes = loadJson(productTypesFilename) if not productTypes: return None if not productTypes.get('@graph'): return None if len(productTypes['@graph']) == 0: return None if not productTypes['@graph'][0].get('rdfs:label'): return None languageExists = False for label in productTypes['@graph'][0]['rdfs:label']: if not label.get('@language'): continue if productTypes['@graph'][0]['rdfs:label']['@language'] == \ systemLanguage: languageExists = True break if not languageExists: print('productTypes ontology does not contain the language ' + systemLanguage) return None dfcIds = {} for item in productTypes['@graph']: if not item.get('@id'): continue if not item.get('rdfs:label'): continue for label in item['rdfs:label']: if not label.get('@language'): continue if not label.get('@value'): continue if label['@language'] == systemLanguage: dfcIds[label['@value'].lower()] = item['@id'] break return dfcIds def getValidSharedItemID(actor: str, displayName: str) -> str: """Removes any invalid characters from the display name to produce an item ID """ removeChars = (' ', '\n', '\r') for ch in removeChars: displayName = displayName.replace(ch, '') removeChars2 = ('+', '/', '\\', '?', '&') for ch in removeChars2: displayName = displayName.replace(ch, '-') displayName = displayName.replace('.', '_') displayName = displayName.replace("’", "'") return actor + '/item/' + displayName def removeSharedItem(baseDir: str, nickname: str, domain: str, displayName: str, httpPrefix: str, domainFull: str) -> None: """Removes a share for a person """ sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' if not os.path.isfile(sharesFilename): print('ERROR: missing shares.json ' + sharesFilename) return sharesJson = loadJson(sharesFilename) if not sharesJson: print('ERROR: shares.json could not be loaded from ' + sharesFilename) return actor = httpPrefix + '://' + domainFull + '/users/' + nickname itemID = getValidSharedItemID(actor, displayName) if sharesJson.get(itemID): # remove any image for the item itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID if sharesJson[itemID]['imageUrl']: formats = getImageExtensions() for ext in formats: if sharesJson[itemID]['imageUrl'].endswith('.' + ext): if os.path.isfile(itemIDfile + '.' + ext): os.remove(itemIDfile + '.' + ext) # remove the item itself del sharesJson[itemID] saveJson(sharesJson, sharesFilename) else: print('ERROR: share index "' + itemID + '" does not exist in ' + sharesFilename) def _addShareDurationSec(duration: str, published: str) -> int: """Returns the duration for the shared item in seconds """ if ' ' not in duration: return 0 durationList = duration.split(' ') if not durationList[0].isdigit(): return 0 if 'hour' in durationList[1]: return published + (int(durationList[0]) * 60 * 60) if 'day' in durationList[1]: return published + (int(durationList[0]) * 60 * 60 * 24) if 'week' in durationList[1]: return published + (int(durationList[0]) * 60 * 60 * 24 * 7) if 'month' in durationList[1]: return published + (int(durationList[0]) * 60 * 60 * 24 * 30) if 'year' in durationList[1]: return published + (int(durationList[0]) * 60 * 60 * 24 * 365) return 0 def _getshareDfcId(baseDir: str, systemLanguage: str, itemType: str, itemCategory: str, translate: {}) -> str: """Attempts to obtain a DFC Id for the shared item, based upon productTypes ontology. See https://github.com/datafoodconsortium/ontology """ if translate['food'] not in itemCategory.lower(): return '' dfcIds = _loadDfcIds(baseDir, systemLanguage) if not dfcIds: return '' itemTypeLower = itemType.lower() matchName = '' matchId = '' for name, uri in dfcIds.items(): if name not in itemTypeLower: continue if len(name) > len(matchName): matchName = name matchId = uri if not matchId: # bag of words match maxMatchedWords = 0 for name, uri in dfcIds.items(): words = name.split(' ') score = 0 for wrd in words: if wrd in itemTypeLower: score += 1 if score > maxMatchedWords: maxMatchedWords = score matchId = uri return matchId def _indicateNewShareAvailable(baseDir: str, httpPrefix: str, domainFull: str) -> None: """Indicate to each account that a new share is available """ for subdir, dirs, files in os.walk(baseDir + '/accounts'): for handle in dirs: if not isAccountDir(handle): continue accountDir = baseDir + '/accounts/' + handle newShareFile = accountDir + '/.newShare' if os.path.isfile(newShareFile): continue nickname = handle.split('@')[0] try: with open(newShareFile, 'w+') as fp: fp.write(httpPrefix + '://' + domainFull + '/users/' + nickname + '/tlshares') except BaseException: pass break def addShare(baseDir: str, httpPrefix: str, nickname: str, domain: str, port: int, displayName: str, summary: str, imageFilename: str, itemQty: int, itemType: str, itemCategory: str, location: str, duration: str, debug: bool, city: str, price: str, currency: str, systemLanguage: str, translate: {}) -> None: """Adds a new share """ sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' sharesJson = {} if os.path.isfile(sharesFilename): sharesJson = loadJson(sharesFilename) duration = duration.lower() published = int(time.time()) durationSec = _addShareDurationSec(duration, published) domainFull = getFullDomain(domain, port) actor = httpPrefix + '://' + domainFull + '/users/' + nickname itemID = getValidSharedItemID(actor, displayName) dfcId = _getshareDfcId(baseDir, systemLanguage, itemType, itemCategory, translate) # has an image for this share been uploaded? imageUrl = None moveImage = False if not imageFilename: sharesImageFilename = \ acctDir(baseDir, nickname, domain) + '/upload' formats = getImageExtensions() for ext in formats: if os.path.isfile(sharesImageFilename + '.' + ext): imageFilename = sharesImageFilename + '.' + ext moveImage = True domainFull = getFullDomain(domain, port) # copy or move the image for the shared item to its destination if imageFilename: if os.path.isfile(imageFilename): if not os.path.isdir(baseDir + '/sharefiles'): os.mkdir(baseDir + '/sharefiles') if not os.path.isdir(baseDir + '/sharefiles/' + nickname): os.mkdir(baseDir + '/sharefiles/' + nickname) itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID formats = getImageExtensions() for ext in formats: if not imageFilename.endswith('.' + ext): continue processMetaData(baseDir, nickname, domain, imageFilename, itemIDfile + '.' + ext, city) if moveImage: os.remove(imageFilename) imageUrl = \ httpPrefix + '://' + domainFull + \ '/sharefiles/' + nickname + '/' + itemID + '.' + ext sharesJson[itemID] = { "displayName": displayName, "summary": summary, "imageUrl": imageUrl, "itemQty": itemQty, "dfcId": dfcId, "itemType": itemType, "category": itemCategory, "location": location, "published": published, "expire": durationSec, "price": price, "currency": currency } saveJson(sharesJson, sharesFilename) _indicateNewShareAvailable(baseDir, httpPrefix, domainFull) def expireShares(baseDir: str) -> None: """Removes expired items from shares """ for subdir, dirs, files in os.walk(baseDir + '/accounts'): for account in dirs: if not isAccountDir(account): continue nickname = account.split('@')[0] domain = account.split('@')[1] _expireSharesForAccount(baseDir, nickname, domain) break def _expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None: """Removes expired items from shares for a particular account """ handleDomain = removeDomainPort(domain) handle = nickname + '@' + handleDomain sharesFilename = baseDir + '/accounts/' + handle + '/shares.json' if not os.path.isfile(sharesFilename): return sharesJson = loadJson(sharesFilename) if not sharesJson: return currTime = int(time.time()) deleteItemID = [] for itemID, item in sharesJson.items(): if currTime > item['expire']: deleteItemID.append(itemID) if not deleteItemID: return for itemID in deleteItemID: del sharesJson[itemID] # remove any associated images itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID formats = getImageExtensions() for ext in formats: if os.path.isfile(itemIDfile + '.' + ext): os.remove(itemIDfile + '.' + ext) saveJson(sharesJson, sharesFilename) def getSharesFeedForPerson(baseDir: str, domain: str, port: int, path: str, httpPrefix: str, sharesPerPage=12) -> {}: """Returns the shares for an account from GET requests """ if '/shares' not in path: return None # handle page numbers headerOnly = True pageNumber = None if '?page=' in path: pageNumber = path.split('?page=')[1] if pageNumber == 'true': pageNumber = 1 else: try: pageNumber = int(pageNumber) except BaseException: pass path = path.split('?page=')[0] headerOnly = False if not path.endswith('/shares'): return None nickname = None if path.startswith('/users/'): nickname = path.replace('/users/', '', 1).replace('/shares', '') if path.startswith('/@'): nickname = path.replace('/@', '', 1).replace('/shares', '') if not nickname: return None if not validNickname(domain, nickname): return None domain = getFullDomain(domain, port) handleDomain = removeDomainPort(domain) sharesFilename = acctDir(baseDir, nickname, handleDomain) + '/shares.json' if headerOnly: noOfShares = 0 if os.path.isfile(sharesFilename): sharesJson = loadJson(sharesFilename) if sharesJson: noOfShares = len(sharesJson.items()) idStr = httpPrefix + '://' + domain + '/users/' + nickname shares = { '@context': 'https://www.w3.org/ns/activitystreams', 'first': idStr + '/shares?page=1', 'id': idStr + '/shares', 'totalItems': str(noOfShares), 'type': 'OrderedCollection' } return shares if not pageNumber: pageNumber = 1 nextPageNumber = int(pageNumber + 1) idStr = httpPrefix + '://' + domain + '/users/' + nickname shares = { '@context': 'https://www.w3.org/ns/activitystreams', 'id': idStr + '/shares?page=' + str(pageNumber), 'orderedItems': [], 'partOf': idStr + '/shares', 'totalItems': 0, 'type': 'OrderedCollectionPage' } if not os.path.isfile(sharesFilename): print("test5") return shares currPage = 1 pageCtr = 0 totalCtr = 0 sharesJson = loadJson(sharesFilename) if sharesJson: for itemID, item in sharesJson.items(): pageCtr += 1 totalCtr += 1 if currPage == pageNumber: shares['orderedItems'].append(item) if pageCtr >= sharesPerPage: pageCtr = 0 currPage += 1 shares['totalItems'] = totalCtr lastPage = int(totalCtr / sharesPerPage) if lastPage < 1: lastPage = 1 if nextPageNumber > lastPage: shares['next'] = \ httpPrefix + '://' + domain + '/users/' + nickname + \ '/shares?page=' + str(lastPage) return shares def sendShareViaServer(baseDir, session, fromNickname: str, password: str, fromDomain: str, fromPort: int, httpPrefix: str, displayName: str, summary: str, imageFilename: str, itemQty: int, itemType: str, itemCategory: str, location: str, duration: str, cachedWebfingers: {}, personCache: {}, debug: bool, projectVersion: str, itemPrice: str, itemCurrency: str) -> {}: """Creates an item share via c2s """ if not session: print('WARN: No session for sendShareViaServer') return 6 fromDomainFull = getFullDomain(fromDomain, fromPort) toUrl = 'https://www.w3.org/ns/activitystreams#Public' ccUrl = httpPrefix + '://' + fromDomainFull + \ '/users/' + fromNickname + '/followers' actor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname newShareJson = { "@context": "https://www.w3.org/ns/activitystreams", 'type': 'Add', 'actor': actor, 'target': actor + '/shares', 'object': { "type": "Offer", "displayName": displayName, "summary": summary, "itemQty": itemQty, "itemType": itemType, "category": itemCategory, "location": location, "duration": duration, "itemPrice": itemPrice, "itemCurrency": itemCurrency, 'to': [toUrl], 'cc': [ccUrl] }, 'to': [toUrl], 'cc': [ccUrl] } handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: share webfinger failed for ' + handle) return 1 if not isinstance(wfRequest, dict): print('WARN: share webfinger for ' + handle + ' did not return a dict. ' + str(wfRequest)) return 1 postToBox = 'outbox' # get the actor inbox for the To handle (inboxUrl, pubKeyId, pubKey, fromPersonId, sharedInbox, avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest, personCache, projectVersion, httpPrefix, fromNickname, fromDomain, postToBox, 83653) if not inboxUrl: if debug: print('DEBUG: share no ' + postToBox + ' was found for ' + handle) return 3 if not fromPersonId: if debug: print('DEBUG: share no actor was found for ' + handle) return 4 authHeader = createBasicAuthHeader(fromNickname, password) if imageFilename: headers = { 'host': fromDomain, 'Authorization': authHeader } postResult = \ postImage(session, imageFilename, [], inboxUrl.replace('/' + postToBox, '/shares'), headers) headers = { 'host': fromDomain, 'Content-type': 'application/json', 'Authorization': authHeader } postResult = \ postJson(httpPrefix, fromDomainFull, session, newShareJson, [], inboxUrl, headers, 30, True) if not postResult: if debug: print('DEBUG: POST share failed for c2s to ' + inboxUrl) # return 5 if debug: print('DEBUG: c2s POST share item success') return newShareJson def sendUndoShareViaServer(baseDir: str, session, fromNickname: str, password: str, fromDomain: str, fromPort: int, httpPrefix: str, displayName: str, cachedWebfingers: {}, personCache: {}, debug: bool, projectVersion: str) -> {}: """Undoes a share via c2s """ if not session: print('WARN: No session for sendUndoShareViaServer') return 6 fromDomainFull = getFullDomain(fromDomain, fromPort) toUrl = 'https://www.w3.org/ns/activitystreams#Public' ccUrl = httpPrefix + '://' + fromDomainFull + \ '/users/' + fromNickname + '/followers' actor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname undoShareJson = { "@context": "https://www.w3.org/ns/activitystreams", 'type': 'Remove', 'actor': actor, 'target': actor + '/shares', 'object': { "type": "Offer", "displayName": displayName, 'to': [toUrl], 'cc': [ccUrl] }, 'to': [toUrl], 'cc': [ccUrl] } handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname # lookup the inbox for the To handle wfRequest = \ webfingerHandle(session, handle, httpPrefix, cachedWebfingers, fromDomain, projectVersion, debug) if not wfRequest: if debug: print('DEBUG: unshare webfinger failed for ' + handle) return 1 if not isinstance(wfRequest, dict): print('WARN: unshare webfinger for ' + handle + ' did not return a dict. ' + str(wfRequest)) return 1 postToBox = 'outbox' # get the actor inbox for the To handle (inboxUrl, pubKeyId, pubKey, fromPersonId, sharedInbox, avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest, personCache, projectVersion, httpPrefix, fromNickname, fromDomain, postToBox, 12663) if not inboxUrl: if debug: print('DEBUG: unshare no ' + postToBox + ' was found for ' + handle) return 3 if not fromPersonId: if debug: print('DEBUG: unshare no actor was found for ' + handle) return 4 authHeader = createBasicAuthHeader(fromNickname, password) headers = { 'host': fromDomain, 'Content-type': 'application/json', 'Authorization': authHeader } postResult = \ postJson(httpPrefix, fromDomainFull, session, undoShareJson, [], inboxUrl, headers, 30, True) if not postResult: if debug: print('DEBUG: POST unshare failed for c2s to ' + inboxUrl) # return 5 if debug: print('DEBUG: c2s POST unshare success') return undoShareJson def outboxShareUpload(baseDir: str, httpPrefix: str, nickname: str, domain: str, port: int, messageJson: {}, debug: bool, city: str, systemLanguage: str, translate: {}) -> None: """ When a shared item is received by the outbox from c2s """ if not messageJson.get('type'): return if not messageJson['type'] == 'Add': return if not hasObjectDict(messageJson): return if not messageJson['object'].get('type'): if debug: print('DEBUG: undo block - no type') return if not messageJson['object']['type'] == 'Offer': if debug: print('DEBUG: not an Offer activity') return if not messageJson['object'].get('displayName'): if debug: print('DEBUG: displayName missing from Offer') return if not messageJson['object'].get('summary'): if debug: print('DEBUG: summary missing from Offer') return if not messageJson['object'].get('itemQty'): if debug: print('DEBUG: itemQty missing from Offer') return if not messageJson['object'].get('itemType'): if debug: print('DEBUG: itemType missing from Offer') return if not messageJson['object'].get('category'): if debug: print('DEBUG: category missing from Offer') return if not messageJson['object'].get('location'): if debug: print('DEBUG: location missing from Offer') return if not messageJson['object'].get('duration'): if debug: print('DEBUG: duration missing from Offer') return addShare(baseDir, httpPrefix, nickname, domain, port, messageJson['object']['displayName'], messageJson['object']['summary'], messageJson['object']['imageFilename'], messageJson['object']['itemQty'], messageJson['object']['itemType'], messageJson['object']['itemCategory'], messageJson['object']['location'], messageJson['object']['duration'], debug, city, messageJson['object']['itemPrice'], messageJson['object']['itemCurrency'], systemLanguage, translate) if debug: print('DEBUG: shared item received via c2s') def outboxUndoShareUpload(baseDir: str, httpPrefix: str, nickname: str, domain: str, port: int, messageJson: {}, debug: bool) -> None: """ When a shared item is removed via c2s """ if not messageJson.get('type'): return if not messageJson['type'] == 'Remove': return if not hasObjectDict(messageJson): return if not messageJson['object'].get('type'): if debug: print('DEBUG: undo block - no type') return if not messageJson['object']['type'] == 'Offer': if debug: print('DEBUG: not an Offer activity') return if not messageJson['object'].get('displayName'): if debug: print('DEBUG: displayName missing from Offer') return domainFull = getFullDomain(domain, port) removeSharedItem(baseDir, nickname, domain, messageJson['object']['displayName'], httpPrefix, domainFull) if debug: print('DEBUG: shared item removed via c2s') def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str, nickname: str, domain: str, domainFull: str, path: str) -> {}: """Returns the endpoint for the shares catalog of a particular account See https://github.com/datafoodconsortium/ontology """ dfcUrl = \ "http://static.datafoodconsortium.org/ontologies/DFC_FullModel.owl#" dfcPtUrl = \ "http://static.datafoodconsortium.org/data/productTypes.rdf#" owner = httpPrefix + '://' + domainFull + '/users/' + nickname dfcInstanceId = owner + '/catalog' endpoint = { "@context": { "DFC": dfcUrl, "dfc-pt": dfcPtUrl, "@base": "http://maPlateformeNationale" }, "@id": dfcInstanceId, "@type": "DFC:Entreprise", "DFC:supplies": [] } sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json' if not os.path.isfile(sharesFilename): return endpoint sharesJson = loadJson(sharesFilename) if not sharesJson: return endpoint for itemID, item in sharesJson.items(): if not item.get('dfcId'): continue if '#' not in item['dfcId']: continue expireDate = datetime.datetime.fromtimestamp(item['durationSec']) expireDateStr = expireDate.strftime("%Y-%m-%dT%H:%M:%SZ") shareId = getValidSharedItemID(owner, item['displayName']) dfcId = item['dfcId'].split('#')[1] catalogItem = { "@id": shareId, "@type": "DFC:SuppliedProduct", "DFC:hasType": "dfc-pt:" + dfcId, "DFC:offeredThrough": owner, "DFC:startDate": item['published'], "DFC:expiryDate": expireDateStr, "DFC:quantity": item['itemQty'], "DFC:totalTheoriticalStock": item['itemQty'], "DFC:price": "0", "DFC:Image": item['imageUrl'], "DFC:description": item['displayName'] + ': ' + item['summary'] } endpoint['DFC:supplies'].append(catalogItem) return endpoint def sharesCatalogEndpoint(baseDir: str, httpPrefix: str, domainFull: str, path: str) -> {}: """Returns the endpoint for the shares catalog for the instance See https://github.com/datafoodconsortium/ontology """ dfcUrl = \ "http://static.datafoodconsortium.org/ontologies/DFC_FullModel.owl#" dfcPtUrl = \ "http://static.datafoodconsortium.org/data/productTypes.rdf#" dfcInstanceId = httpPrefix + '://' + domainFull + '/catalog' endpoint = { "@context": { "DFC": dfcUrl, "dfc-pt": dfcPtUrl, "@base": "http://maPlateformeNationale" }, "@id": dfcInstanceId, "@type": "DFC:Entreprise", "DFC:supplies": [] } for subdir, dirs, files in os.walk(baseDir + '/accounts'): for acct in dirs: if not isAccountDir(acct): continue nickname = acct.split('@')[0] domain = acct.split('@')[1] owner = httpPrefix + '://' + domainFull + '/users/' + nickname sharesFilename = \ acctDir(baseDir, nickname, domain) + '/shares.json' if not os.path.isfile(sharesFilename): continue sharesJson = loadJson(sharesFilename) if not sharesJson: continue for itemID, item in sharesJson.items(): if not item.get('dfcId'): continue if '#' not in item['dfcId']: continue expireDate = \ datetime.datetime.fromtimestamp(item['durationSec']) expireDateStr = expireDate.strftime("%Y-%m-%dT%H:%M:%SZ") description = item['displayName'] + ': ' + item['summary'] shareId = getValidSharedItemID(owner, item['displayName']) dfcId = item['dfcId'].split('#')[1] catalogItem = { "@id": shareId, "@type": "DFC:SuppliedProduct", "DFC:hasType": "dfc-pt:" + dfcId, "DFC:offeredThrough": owner, "DFC:startDate": item['published'], "DFC:expiryDate": expireDateStr, "DFC:quantity": item['itemQty'], "DFC:totalTheoriticalStock": item['itemQty'], "DFC:price": "0", "DFC:Image": item['imageUrl'], "DFC:description": description } endpoint['DFC:supplies'].append(catalogItem) return endpoint