mirror of https://gitlab.com/bashrc2/epicyon
1529 lines
54 KiB
Python
1529 lines
54 KiB
Python
__filename__ = "shares.py"
|
||
__author__ = "Bob Mottram"
|
||
__license__ = "AGPL3+"
|
||
__version__ = "1.2.0"
|
||
__maintainer__ = "Bob Mottram"
|
||
__email__ = "bob@freedombone.net"
|
||
__status__ = "Production"
|
||
__module_group__ = "Timeline"
|
||
|
||
import os
|
||
import re
|
||
import secrets
|
||
import time
|
||
import datetime
|
||
from random import randint
|
||
from pprint import pprint
|
||
from session import getJson
|
||
from webfinger import webfingerHandle
|
||
from auth import createBasicAuthHeader
|
||
from auth import constantTimeStringCheck
|
||
from posts import getPersonBox
|
||
from session import postJson
|
||
from session import postImage
|
||
from session import createSession
|
||
from utils import dateStringToSeconds
|
||
from utils import dateSecondsToString
|
||
from utils import getConfigParam
|
||
from utils import getFullDomain
|
||
from utils import validNickname
|
||
from utils import loadJson
|
||
from utils import saveJson
|
||
from utils import getImageExtensions
|
||
from utils import hasObjectDict
|
||
from utils import removeDomainPort
|
||
from utils import isAccountDir
|
||
from utils import acctDir
|
||
from utils import isfloat
|
||
from utils import getCategoryTypes
|
||
from media import processMetaData
|
||
from filters import isFilteredGlobally
|
||
from siteactive import siteIsActive
|
||
from content import getPriceFromString
|
||
|
||
|
||
def _loadDfcIds(baseDir: str, systemLanguage: str,
|
||
productType: str) -> {}:
|
||
"""Loads the product types ontology
|
||
This is used to add an id to shared items
|
||
"""
|
||
productTypesFilename = \
|
||
baseDir + '/ontology/custom' + productType.title() + 'Types.json'
|
||
if not os.path.isfile(productTypesFilename):
|
||
productTypesFilename = \
|
||
baseDir + '/ontology/' + productType + 'Types.json'
|
||
productTypes = loadJson(productTypesFilename)
|
||
if not productTypes:
|
||
print('Unable to load ontology: ' + productTypesFilename)
|
||
return None
|
||
if not productTypes.get('@graph'):
|
||
print('No @graph list within ontology')
|
||
return None
|
||
if len(productTypes['@graph']) == 0:
|
||
print('@graph list has no contents')
|
||
return None
|
||
if not productTypes['@graph'][0].get('rdfs:label'):
|
||
print('@graph list entry has no rdfs:label')
|
||
return None
|
||
languageExists = False
|
||
for label in productTypes['@graph'][0]['rdfs:label']:
|
||
if not label.get('@language'):
|
||
continue
|
||
if label['@language'] == systemLanguage:
|
||
languageExists = True
|
||
break
|
||
if not languageExists:
|
||
print('productTypes ontology does not contain the language ' +
|
||
systemLanguage)
|
||
return None
|
||
dfcIds = {}
|
||
for item in productTypes['@graph']:
|
||
if not item.get('@id'):
|
||
continue
|
||
if not item.get('rdfs:label'):
|
||
continue
|
||
for label in item['rdfs:label']:
|
||
if not label.get('@language'):
|
||
continue
|
||
if not label.get('@value'):
|
||
continue
|
||
if label['@language'] == systemLanguage:
|
||
dfcIds[label['@value'].lower()] = item['@id']
|
||
break
|
||
return dfcIds
|
||
|
||
|
||
def _getValidSharedItemID(actor: str, displayName: str) -> str:
|
||
"""Removes any invalid characters from the display name to
|
||
produce an item ID
|
||
"""
|
||
removeChars = (' ', '\n', '\r', '#')
|
||
for ch in removeChars:
|
||
displayName = displayName.replace(ch, '')
|
||
removeChars2 = ('+', '/', '\\', '?', '&')
|
||
for ch in removeChars2:
|
||
displayName = displayName.replace(ch, '-')
|
||
displayName = displayName.replace('.', '_')
|
||
displayName = displayName.replace("’", "'")
|
||
actor = actor.replace('://', '___')
|
||
actor = actor.replace('/', '--')
|
||
return actor + '--shareditems--' + displayName
|
||
|
||
|
||
def removeSharedItem(baseDir: str, nickname: str, domain: str,
|
||
itemID: str,
|
||
httpPrefix: str, domainFull: str) -> None:
|
||
"""Removes a share for a person
|
||
"""
|
||
sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json'
|
||
if not os.path.isfile(sharesFilename):
|
||
print('ERROR: missing shares.json ' + sharesFilename)
|
||
return
|
||
|
||
sharesJson = loadJson(sharesFilename)
|
||
if not sharesJson:
|
||
print('ERROR: shares.json could not be loaded from ' + sharesFilename)
|
||
return
|
||
|
||
if sharesJson.get(itemID):
|
||
# remove any image for the item
|
||
itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID
|
||
if sharesJson[itemID]['imageUrl']:
|
||
formats = getImageExtensions()
|
||
for ext in formats:
|
||
if sharesJson[itemID]['imageUrl'].endswith('.' + ext):
|
||
if os.path.isfile(itemIDfile + '.' + ext):
|
||
os.remove(itemIDfile + '.' + ext)
|
||
# remove the item itself
|
||
del sharesJson[itemID]
|
||
saveJson(sharesJson, sharesFilename)
|
||
else:
|
||
print('ERROR: share index "' + itemID +
|
||
'" does not exist in ' + sharesFilename)
|
||
|
||
|
||
def _addShareDurationSec(duration: str, published: int) -> int:
|
||
"""Returns the duration for the shared item in seconds
|
||
"""
|
||
if ' ' not in duration:
|
||
return 0
|
||
durationList = duration.split(' ')
|
||
if not durationList[0].isdigit():
|
||
return 0
|
||
if 'hour' in durationList[1]:
|
||
return published + (int(durationList[0]) * 60 * 60)
|
||
if 'day' in durationList[1]:
|
||
return published + (int(durationList[0]) * 60 * 60 * 24)
|
||
if 'week' in durationList[1]:
|
||
return published + (int(durationList[0]) * 60 * 60 * 24 * 7)
|
||
if 'month' in durationList[1]:
|
||
return published + (int(durationList[0]) * 60 * 60 * 24 * 30)
|
||
if 'year' in durationList[1]:
|
||
return published + (int(durationList[0]) * 60 * 60 * 24 * 365)
|
||
return 0
|
||
|
||
|
||
def _dfcProductTypeFromCategory(baseDir: str,
|
||
itemCategory: str, translate: {}) -> str:
|
||
"""Does the shared item category match a DFC product type?
|
||
If so then return the product type.
|
||
This will be used to select an appropriate ontology file
|
||
such as ontology/foodTypes.json
|
||
"""
|
||
productTypesList = getCategoryTypes(baseDir)
|
||
categoryLower = itemCategory.lower()
|
||
for productType in productTypesList:
|
||
if translate.get(productType):
|
||
if translate[productType] in categoryLower:
|
||
return productType
|
||
else:
|
||
if productType in categoryLower:
|
||
return productType
|
||
return None
|
||
|
||
|
||
def _getshareDfcId(baseDir: str, systemLanguage: str,
|
||
itemType: str, itemCategory: str,
|
||
translate: {}, dfcIds: {} = None) -> str:
|
||
"""Attempts to obtain a DFC Id for the shared item,
|
||
based upon productTypes ontology.
|
||
See https://github.com/datafoodconsortium/ontology
|
||
"""
|
||
# does the category field match any prodyct type ontology
|
||
# files in the ontology subdirectory?
|
||
matchedProductType = \
|
||
_dfcProductTypeFromCategory(baseDir, itemCategory, translate)
|
||
if not matchedProductType:
|
||
itemType = itemType.replace(' ', '_')
|
||
itemType = itemType.replace('.', '')
|
||
return 'epicyon#' + itemType
|
||
if not dfcIds:
|
||
dfcIds = _loadDfcIds(baseDir, systemLanguage, matchedProductType)
|
||
if not dfcIds:
|
||
return ''
|
||
itemTypeLower = itemType.lower()
|
||
matchName = ''
|
||
matchId = ''
|
||
for name, uri in dfcIds.items():
|
||
if name not in itemTypeLower:
|
||
continue
|
||
if len(name) > len(matchName):
|
||
matchName = name
|
||
matchId = uri
|
||
if not matchId:
|
||
# bag of words match
|
||
maxMatchedWords = 0
|
||
for name, uri in dfcIds.items():
|
||
name = name.replace('-', ' ')
|
||
words = name.split(' ')
|
||
score = 0
|
||
for wrd in words:
|
||
if wrd in itemTypeLower:
|
||
score += 1
|
||
if score > maxMatchedWords:
|
||
maxMatchedWords = score
|
||
matchId = uri
|
||
return matchId
|
||
|
||
|
||
def _getshareTypeFromDfcId(dfcUri: str, dfcIds: {}) -> str:
|
||
"""Attempts to obtain a share item type from its DFC Id,
|
||
based upon productTypes ontology.
|
||
See https://github.com/datafoodconsortium/ontology
|
||
"""
|
||
if dfcUri.startswith('epicyon#'):
|
||
itemType = dfcUri.split('#')[1]
|
||
itemType = itemType.replace('_', ' ')
|
||
return itemType
|
||
|
||
for name, uri in dfcIds.items():
|
||
if uri.endswith('#' + dfcUri):
|
||
return name
|
||
elif uri == dfcUri:
|
||
return name
|
||
return None
|
||
|
||
|
||
def _indicateNewShareAvailable(baseDir: str, httpPrefix: str,
|
||
domainFull: str) -> None:
|
||
"""Indicate to each account that a new share is available
|
||
"""
|
||
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
|
||
for handle in dirs:
|
||
if not isAccountDir(handle):
|
||
continue
|
||
accountDir = baseDir + '/accounts/' + handle
|
||
newShareFile = accountDir + '/.newShare'
|
||
if os.path.isfile(newShareFile):
|
||
continue
|
||
nickname = handle.split('@')[0]
|
||
try:
|
||
with open(newShareFile, 'w+') as fp:
|
||
fp.write(httpPrefix + '://' + domainFull +
|
||
'/users/' + nickname + '/tlshares')
|
||
except BaseException:
|
||
pass
|
||
break
|
||
|
||
|
||
def addShare(baseDir: str,
|
||
httpPrefix: str, nickname: str, domain: str, port: int,
|
||
displayName: str, summary: str, imageFilename: str,
|
||
itemQty: float, itemType: str, itemCategory: str, location: str,
|
||
duration: str, debug: bool, city: str,
|
||
price: str, currency: str,
|
||
systemLanguage: str, translate: {}) -> None:
|
||
"""Adds a new share
|
||
"""
|
||
if isFilteredGlobally(baseDir,
|
||
displayName + ' ' + summary + ' ' +
|
||
itemType + ' ' + itemCategory):
|
||
print('Shared item was filtered due to content')
|
||
return
|
||
sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json'
|
||
sharesJson = {}
|
||
if os.path.isfile(sharesFilename):
|
||
sharesJson = loadJson(sharesFilename, 1, 2)
|
||
|
||
duration = duration.lower()
|
||
published = int(time.time())
|
||
durationSec = _addShareDurationSec(duration, published)
|
||
|
||
domainFull = getFullDomain(domain, port)
|
||
actor = httpPrefix + '://' + domainFull + '/users/' + nickname
|
||
itemID = _getValidSharedItemID(actor, displayName)
|
||
dfcId = _getshareDfcId(baseDir, systemLanguage,
|
||
itemType, itemCategory, translate)
|
||
|
||
# has an image for this share been uploaded?
|
||
imageUrl = None
|
||
moveImage = False
|
||
if not imageFilename:
|
||
sharesImageFilename = \
|
||
acctDir(baseDir, nickname, domain) + '/upload'
|
||
formats = getImageExtensions()
|
||
for ext in formats:
|
||
if os.path.isfile(sharesImageFilename + '.' + ext):
|
||
imageFilename = sharesImageFilename + '.' + ext
|
||
moveImage = True
|
||
|
||
domainFull = getFullDomain(domain, port)
|
||
|
||
# copy or move the image for the shared item to its destination
|
||
if imageFilename:
|
||
if os.path.isfile(imageFilename):
|
||
if not os.path.isdir(baseDir + '/sharefiles'):
|
||
os.mkdir(baseDir + '/sharefiles')
|
||
if not os.path.isdir(baseDir + '/sharefiles/' + nickname):
|
||
os.mkdir(baseDir + '/sharefiles/' + nickname)
|
||
itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID
|
||
formats = getImageExtensions()
|
||
for ext in formats:
|
||
if not imageFilename.endswith('.' + ext):
|
||
continue
|
||
processMetaData(baseDir, nickname, domain,
|
||
imageFilename, itemIDfile + '.' + ext,
|
||
city)
|
||
if moveImage:
|
||
os.remove(imageFilename)
|
||
imageUrl = \
|
||
httpPrefix + '://' + domainFull + \
|
||
'/sharefiles/' + nickname + '/' + itemID + '.' + ext
|
||
|
||
sharesJson[itemID] = {
|
||
"displayName": displayName,
|
||
"summary": summary,
|
||
"imageUrl": imageUrl,
|
||
"itemQty": float(itemQty),
|
||
"dfcId": dfcId,
|
||
"itemType": itemType,
|
||
"category": itemCategory,
|
||
"location": location,
|
||
"published": published,
|
||
"expire": durationSec,
|
||
"itemPrice": price,
|
||
"itemCurrency": currency
|
||
}
|
||
|
||
saveJson(sharesJson, sharesFilename)
|
||
|
||
_indicateNewShareAvailable(baseDir, httpPrefix, domainFull)
|
||
|
||
|
||
def expireShares(baseDir: str) -> None:
|
||
"""Removes expired items from shares
|
||
"""
|
||
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
|
||
for account in dirs:
|
||
if not isAccountDir(account):
|
||
continue
|
||
nickname = account.split('@')[0]
|
||
domain = account.split('@')[1]
|
||
_expireSharesForAccount(baseDir, nickname, domain)
|
||
break
|
||
|
||
|
||
def _expireSharesForAccount(baseDir: str, nickname: str, domain: str) -> None:
|
||
"""Removes expired items from shares for a particular account
|
||
"""
|
||
handleDomain = removeDomainPort(domain)
|
||
handle = nickname + '@' + handleDomain
|
||
sharesFilename = baseDir + '/accounts/' + handle + '/shares.json'
|
||
if not os.path.isfile(sharesFilename):
|
||
return
|
||
sharesJson = loadJson(sharesFilename, 1, 2)
|
||
if not sharesJson:
|
||
return
|
||
currTime = int(time.time())
|
||
deleteItemID = []
|
||
for itemID, item in sharesJson.items():
|
||
if currTime > item['expire']:
|
||
deleteItemID.append(itemID)
|
||
if not deleteItemID:
|
||
return
|
||
for itemID in deleteItemID:
|
||
del sharesJson[itemID]
|
||
# remove any associated images
|
||
itemIDfile = baseDir + '/sharefiles/' + nickname + '/' + itemID
|
||
formats = getImageExtensions()
|
||
for ext in formats:
|
||
if os.path.isfile(itemIDfile + '.' + ext):
|
||
os.remove(itemIDfile + '.' + ext)
|
||
saveJson(sharesJson, sharesFilename)
|
||
|
||
|
||
def getSharesFeedForPerson(baseDir: str,
|
||
domain: str, port: int,
|
||
path: str, httpPrefix: str,
|
||
sharesPerPage=12) -> {}:
|
||
"""Returns the shares for an account from GET requests
|
||
"""
|
||
if '/shares' not in path:
|
||
return None
|
||
# handle page numbers
|
||
headerOnly = True
|
||
pageNumber = None
|
||
if '?page=' in path:
|
||
pageNumber = path.split('?page=')[1]
|
||
if pageNumber == 'true':
|
||
pageNumber = 1
|
||
else:
|
||
try:
|
||
pageNumber = int(pageNumber)
|
||
except BaseException:
|
||
pass
|
||
path = path.split('?page=')[0]
|
||
headerOnly = False
|
||
|
||
if not path.endswith('/shares'):
|
||
return None
|
||
nickname = None
|
||
if path.startswith('/users/'):
|
||
nickname = path.replace('/users/', '', 1).replace('/shares', '')
|
||
if path.startswith('/@'):
|
||
nickname = path.replace('/@', '', 1).replace('/shares', '')
|
||
if not nickname:
|
||
return None
|
||
if not validNickname(domain, nickname):
|
||
return None
|
||
|
||
domain = getFullDomain(domain, port)
|
||
|
||
handleDomain = removeDomainPort(domain)
|
||
sharesFilename = acctDir(baseDir, nickname, handleDomain) + '/shares.json'
|
||
|
||
if headerOnly:
|
||
noOfShares = 0
|
||
if os.path.isfile(sharesFilename):
|
||
sharesJson = loadJson(sharesFilename)
|
||
if sharesJson:
|
||
noOfShares = len(sharesJson.items())
|
||
idStr = httpPrefix + '://' + domain + '/users/' + nickname
|
||
shares = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'first': idStr + '/shares?page=1',
|
||
'id': idStr + '/shares',
|
||
'totalItems': str(noOfShares),
|
||
'type': 'OrderedCollection'
|
||
}
|
||
return shares
|
||
|
||
if not pageNumber:
|
||
pageNumber = 1
|
||
|
||
nextPageNumber = int(pageNumber + 1)
|
||
idStr = httpPrefix + '://' + domain + '/users/' + nickname
|
||
shares = {
|
||
'@context': 'https://www.w3.org/ns/activitystreams',
|
||
'id': idStr + '/shares?page=' + str(pageNumber),
|
||
'orderedItems': [],
|
||
'partOf': idStr + '/shares',
|
||
'totalItems': 0,
|
||
'type': 'OrderedCollectionPage'
|
||
}
|
||
|
||
if not os.path.isfile(sharesFilename):
|
||
print("test5")
|
||
return shares
|
||
currPage = 1
|
||
pageCtr = 0
|
||
totalCtr = 0
|
||
|
||
sharesJson = loadJson(sharesFilename)
|
||
if sharesJson:
|
||
for itemID, item in sharesJson.items():
|
||
pageCtr += 1
|
||
totalCtr += 1
|
||
if currPage == pageNumber:
|
||
item['shareId'] = itemID
|
||
shares['orderedItems'].append(item)
|
||
if pageCtr >= sharesPerPage:
|
||
pageCtr = 0
|
||
currPage += 1
|
||
shares['totalItems'] = totalCtr
|
||
lastPage = int(totalCtr / sharesPerPage)
|
||
if lastPage < 1:
|
||
lastPage = 1
|
||
if nextPageNumber > lastPage:
|
||
shares['next'] = \
|
||
httpPrefix + '://' + domain + '/users/' + nickname + \
|
||
'/shares?page=' + str(lastPage)
|
||
return shares
|
||
|
||
|
||
def sendShareViaServer(baseDir, session,
|
||
fromNickname: str, password: str,
|
||
fromDomain: str, fromPort: int,
|
||
httpPrefix: str, displayName: str,
|
||
summary: str, imageFilename: str,
|
||
itemQty: float, itemType: str, itemCategory: str,
|
||
location: str, duration: str,
|
||
cachedWebfingers: {}, personCache: {},
|
||
debug: bool, projectVersion: str,
|
||
itemPrice: str, itemCurrency: str) -> {}:
|
||
"""Creates an item share via c2s
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendShareViaServer')
|
||
return 6
|
||
|
||
# convert $4.23 to 4.23 USD
|
||
newItemPrice, newItemCurrency = getPriceFromString(itemPrice)
|
||
if newItemPrice != itemPrice:
|
||
itemPrice = newItemPrice
|
||
if not itemCurrency:
|
||
if newItemCurrency != itemCurrency:
|
||
itemCurrency = newItemCurrency
|
||
|
||
fromDomainFull = getFullDomain(fromDomain, fromPort)
|
||
|
||
toUrl = 'https://www.w3.org/ns/activitystreams#Public'
|
||
ccUrl = httpPrefix + '://' + fromDomainFull + \
|
||
'/users/' + fromNickname + '/followers'
|
||
|
||
actor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
|
||
newShareJson = {
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
'type': 'Add',
|
||
'actor': actor,
|
||
'target': actor + '/shares',
|
||
'object': {
|
||
"type": "Offer",
|
||
"displayName": displayName,
|
||
"summary": summary,
|
||
"itemQty": float(itemQty),
|
||
"itemType": itemType,
|
||
"category": itemCategory,
|
||
"location": location,
|
||
"duration": duration,
|
||
"itemPrice": itemPrice,
|
||
"itemCurrency": itemCurrency,
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
},
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
}
|
||
|
||
handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = \
|
||
webfingerHandle(session, handle, httpPrefix,
|
||
cachedWebfingers,
|
||
fromDomain, projectVersion, debug, False)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: share webfinger failed for ' + handle)
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: share webfinger for ' + handle +
|
||
' did not return a dict. ' + str(wfRequest))
|
||
return 1
|
||
|
||
postToBox = 'outbox'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
fromPersonId, sharedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache, projectVersion,
|
||
httpPrefix, fromNickname,
|
||
fromDomain, postToBox,
|
||
83653)
|
||
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: share no ' + postToBox +
|
||
' was found for ' + handle)
|
||
return 3
|
||
if not fromPersonId:
|
||
if debug:
|
||
print('DEBUG: share no actor was found for ' + handle)
|
||
return 4
|
||
|
||
authHeader = createBasicAuthHeader(fromNickname, password)
|
||
|
||
if imageFilename:
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = \
|
||
postImage(session, imageFilename, [],
|
||
inboxUrl.replace('/' + postToBox, '/shares'),
|
||
headers)
|
||
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = \
|
||
postJson(httpPrefix, fromDomainFull,
|
||
session, newShareJson, [], inboxUrl, headers, 30, True)
|
||
if not postResult:
|
||
if debug:
|
||
print('DEBUG: POST share failed for c2s to ' + inboxUrl)
|
||
# return 5
|
||
|
||
if debug:
|
||
print('DEBUG: c2s POST share item success')
|
||
|
||
return newShareJson
|
||
|
||
|
||
def sendUndoShareViaServer(baseDir: str, session,
|
||
fromNickname: str, password: str,
|
||
fromDomain: str, fromPort: int,
|
||
httpPrefix: str, displayName: str,
|
||
cachedWebfingers: {}, personCache: {},
|
||
debug: bool, projectVersion: str) -> {}:
|
||
"""Undoes a share via c2s
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for sendUndoShareViaServer')
|
||
return 6
|
||
|
||
fromDomainFull = getFullDomain(fromDomain, fromPort)
|
||
|
||
toUrl = 'https://www.w3.org/ns/activitystreams#Public'
|
||
ccUrl = httpPrefix + '://' + fromDomainFull + \
|
||
'/users/' + fromNickname + '/followers'
|
||
|
||
actor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
|
||
undoShareJson = {
|
||
"@context": "https://www.w3.org/ns/activitystreams",
|
||
'type': 'Remove',
|
||
'actor': actor,
|
||
'target': actor + '/shares',
|
||
'object': {
|
||
"type": "Offer",
|
||
"displayName": displayName,
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
},
|
||
'to': [toUrl],
|
||
'cc': [ccUrl]
|
||
}
|
||
|
||
handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
|
||
|
||
# lookup the inbox for the To handle
|
||
wfRequest = \
|
||
webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
|
||
fromDomain, projectVersion, debug, False)
|
||
if not wfRequest:
|
||
if debug:
|
||
print('DEBUG: unshare webfinger failed for ' + handle)
|
||
return 1
|
||
if not isinstance(wfRequest, dict):
|
||
print('WARN: unshare webfinger for ' + handle +
|
||
' did not return a dict. ' + str(wfRequest))
|
||
return 1
|
||
|
||
postToBox = 'outbox'
|
||
|
||
# get the actor inbox for the To handle
|
||
(inboxUrl, pubKeyId, pubKey,
|
||
fromPersonId, sharedInbox,
|
||
avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
|
||
personCache, projectVersion,
|
||
httpPrefix, fromNickname,
|
||
fromDomain, postToBox,
|
||
12663)
|
||
|
||
if not inboxUrl:
|
||
if debug:
|
||
print('DEBUG: unshare no ' + postToBox +
|
||
' was found for ' + handle)
|
||
return 3
|
||
if not fromPersonId:
|
||
if debug:
|
||
print('DEBUG: unshare no actor was found for ' + handle)
|
||
return 4
|
||
|
||
authHeader = createBasicAuthHeader(fromNickname, password)
|
||
|
||
headers = {
|
||
'host': fromDomain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader
|
||
}
|
||
postResult = \
|
||
postJson(httpPrefix, fromDomainFull,
|
||
session, undoShareJson, [], inboxUrl,
|
||
headers, 30, True)
|
||
if not postResult:
|
||
if debug:
|
||
print('DEBUG: POST unshare failed for c2s to ' + inboxUrl)
|
||
# return 5
|
||
|
||
if debug:
|
||
print('DEBUG: c2s POST unshare success')
|
||
|
||
return undoShareJson
|
||
|
||
|
||
def getSharedItemsCatalogViaServer(baseDir, session,
|
||
nickname: str, password: str,
|
||
domain: str, port: int,
|
||
httpPrefix: str, debug: bool) -> {}:
|
||
"""Returns the shared items catalog via c2s
|
||
"""
|
||
if not session:
|
||
print('WARN: No session for getSharedItemsCatalogViaServer')
|
||
return 6
|
||
|
||
authHeader = createBasicAuthHeader(nickname, password)
|
||
|
||
headers = {
|
||
'host': domain,
|
||
'Content-type': 'application/json',
|
||
'Authorization': authHeader,
|
||
'Accept': 'application/json'
|
||
}
|
||
domainFull = getFullDomain(domain, port)
|
||
url = httpPrefix + '://' + domainFull + '/users/' + nickname + '/catalog'
|
||
if debug:
|
||
print('Shared items catalog request to: ' + url)
|
||
catalogJson = getJson(session, url, headers, None, debug,
|
||
__version__, httpPrefix, None)
|
||
if not catalogJson:
|
||
if debug:
|
||
print('DEBUG: GET shared items catalog failed for c2s to ' + url)
|
||
# return 5
|
||
|
||
if debug:
|
||
print('DEBUG: c2s GET shared items catalog success')
|
||
|
||
return catalogJson
|
||
|
||
|
||
def outboxShareUpload(baseDir: str, httpPrefix: str,
|
||
nickname: str, domain: str, port: int,
|
||
messageJson: {}, debug: bool, city: str,
|
||
systemLanguage: str, translate: {}) -> None:
|
||
""" When a shared item is received by the outbox from c2s
|
||
"""
|
||
if not messageJson.get('type'):
|
||
return
|
||
if not messageJson['type'] == 'Add':
|
||
return
|
||
if not hasObjectDict(messageJson):
|
||
return
|
||
if not messageJson['object'].get('type'):
|
||
if debug:
|
||
print('DEBUG: undo block - no type')
|
||
return
|
||
if not messageJson['object']['type'] == 'Offer':
|
||
if debug:
|
||
print('DEBUG: not an Offer activity')
|
||
return
|
||
if not messageJson['object'].get('displayName'):
|
||
if debug:
|
||
print('DEBUG: displayName missing from Offer')
|
||
return
|
||
if not messageJson['object'].get('summary'):
|
||
if debug:
|
||
print('DEBUG: summary missing from Offer')
|
||
return
|
||
if not messageJson['object'].get('itemQty'):
|
||
if debug:
|
||
print('DEBUG: itemQty missing from Offer')
|
||
return
|
||
if not messageJson['object'].get('itemType'):
|
||
if debug:
|
||
print('DEBUG: itemType missing from Offer')
|
||
return
|
||
if not messageJson['object'].get('category'):
|
||
if debug:
|
||
print('DEBUG: category missing from Offer')
|
||
return
|
||
if not messageJson['object'].get('duration'):
|
||
if debug:
|
||
print('DEBUG: duration missing from Offer')
|
||
return
|
||
itemQty = float(messageJson['object']['itemQty'])
|
||
location = ''
|
||
if messageJson['object'].get('location'):
|
||
location = messageJson['object']['location']
|
||
imageFilename = None
|
||
if messageJson['object'].get('imageFilename'):
|
||
imageFilename = messageJson['object']['imageFilename']
|
||
if debug:
|
||
print('Adding shared item')
|
||
pprint(messageJson)
|
||
|
||
addShare(baseDir,
|
||
httpPrefix, nickname, domain, port,
|
||
messageJson['object']['displayName'],
|
||
messageJson['object']['summary'],
|
||
imageFilename,
|
||
itemQty,
|
||
messageJson['object']['itemType'],
|
||
messageJson['object']['category'],
|
||
location,
|
||
messageJson['object']['duration'],
|
||
debug, city,
|
||
messageJson['object']['itemPrice'],
|
||
messageJson['object']['itemCurrency'],
|
||
systemLanguage, translate)
|
||
if debug:
|
||
print('DEBUG: shared item received via c2s')
|
||
|
||
|
||
def outboxUndoShareUpload(baseDir: str, httpPrefix: str,
|
||
nickname: str, domain: str, port: int,
|
||
messageJson: {}, debug: bool) -> None:
|
||
""" When a shared item is removed via c2s
|
||
"""
|
||
if not messageJson.get('type'):
|
||
return
|
||
if not messageJson['type'] == 'Remove':
|
||
return
|
||
if not hasObjectDict(messageJson):
|
||
return
|
||
if not messageJson['object'].get('type'):
|
||
if debug:
|
||
print('DEBUG: undo block - no type')
|
||
return
|
||
if not messageJson['object']['type'] == 'Offer':
|
||
if debug:
|
||
print('DEBUG: not an Offer activity')
|
||
return
|
||
if not messageJson['object'].get('displayName'):
|
||
if debug:
|
||
print('DEBUG: displayName missing from Offer')
|
||
return
|
||
domainFull = getFullDomain(domain, port)
|
||
removeSharedItem(baseDir, nickname, domain,
|
||
messageJson['object']['displayName'],
|
||
httpPrefix, domainFull)
|
||
if debug:
|
||
print('DEBUG: shared item removed via c2s')
|
||
|
||
|
||
def _sharesCatalogParams(path: str) -> (bool, float, float, str):
|
||
"""Returns parameters when accessing the shares catalog
|
||
"""
|
||
today = False
|
||
minPrice = 0
|
||
maxPrice = 9999999
|
||
matchPattern = None
|
||
if '?' not in path:
|
||
return today, minPrice, maxPrice, matchPattern
|
||
args = path.split('?', 1)[1]
|
||
argList = args.split(';')
|
||
for arg in argList:
|
||
if '=' not in arg:
|
||
continue
|
||
key = arg.split('=')[0].lower()
|
||
value = arg.split('=')[1]
|
||
if key == 'today':
|
||
value = value.lower()
|
||
if 't' in value or 'y' in value or '1' in value:
|
||
today = True
|
||
elif key.startswith('min'):
|
||
if isfloat(value):
|
||
minPrice = float(value)
|
||
elif key.startswith('max'):
|
||
if isfloat(value):
|
||
maxPrice = float(value)
|
||
elif key.startswith('match'):
|
||
matchPattern = value
|
||
return today, minPrice, maxPrice, matchPattern
|
||
|
||
|
||
def sharesCatalogAccountEndpoint(baseDir: str, httpPrefix: str,
|
||
nickname: str, domain: str,
|
||
domainFull: str,
|
||
path: str, debug: bool) -> {}:
|
||
"""Returns the endpoint for the shares catalog of a particular account
|
||
See https://github.com/datafoodconsortium/ontology
|
||
"""
|
||
today, minPrice, maxPrice, matchPattern = _sharesCatalogParams(path)
|
||
dfcUrl = \
|
||
"http://static.datafoodconsortium.org/ontologies/DFC_FullModel.owl#"
|
||
dfcPtUrl = \
|
||
"http://static.datafoodconsortium.org/data/productTypes.rdf#"
|
||
owner = httpPrefix + '://' + domainFull + '/users/' + nickname
|
||
dfcInstanceId = owner + '/catalog'
|
||
endpoint = {
|
||
"@context": {
|
||
"DFC": dfcUrl,
|
||
"dfc-pt": dfcPtUrl,
|
||
"@base": "http://maPlateformeNationale"
|
||
},
|
||
"@id": dfcInstanceId,
|
||
"@type": "DFC:Entreprise",
|
||
"DFC:supplies": []
|
||
}
|
||
|
||
currDate = datetime.datetime.utcnow()
|
||
currDateStr = currDate.strftime("%Y-%m-%d")
|
||
|
||
sharesFilename = acctDir(baseDir, nickname, domain) + '/shares.json'
|
||
if not os.path.isfile(sharesFilename):
|
||
if debug:
|
||
print('shares.json file not found: ' + sharesFilename)
|
||
return endpoint
|
||
sharesJson = loadJson(sharesFilename, 1, 2)
|
||
if not sharesJson:
|
||
if debug:
|
||
print('Unable to load json for ' + sharesFilename)
|
||
return endpoint
|
||
|
||
for itemID, item in sharesJson.items():
|
||
if not item.get('dfcId'):
|
||
if debug:
|
||
print('Item does not have dfcId: ' + itemID)
|
||
continue
|
||
if '#' not in item['dfcId']:
|
||
continue
|
||
if today:
|
||
if not item['published'].startswith(currDateStr):
|
||
continue
|
||
if minPrice is not None:
|
||
if float(item['itemPrice']) < minPrice:
|
||
continue
|
||
if maxPrice is not None:
|
||
if float(item['itemPrice']) > maxPrice:
|
||
continue
|
||
description = item['displayName'] + ': ' + item['summary']
|
||
if matchPattern:
|
||
if not re.match(matchPattern, description):
|
||
continue
|
||
|
||
expireDate = datetime.datetime.fromtimestamp(item['expire'])
|
||
expireDateStr = expireDate.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
shareId = _getValidSharedItemID(owner, item['displayName'])
|
||
if item['dfcId'].startswith('epicyon#'):
|
||
dfcId = "epicyon:" + item['dfcId'].split('#')[1]
|
||
else:
|
||
dfcId = "dfc-pt:" + item['dfcId'].split('#')[1]
|
||
priceStr = item['itemPrice'] + ' ' + item['itemCurrency']
|
||
catalogItem = {
|
||
"@id": shareId,
|
||
"@type": "DFC:SuppliedProduct",
|
||
"DFC:hasType": dfcId,
|
||
"DFC:startDate": item['published'],
|
||
"DFC:expiryDate": expireDateStr,
|
||
"DFC:quantity": float(item['itemQty']),
|
||
"DFC:price": priceStr,
|
||
"DFC:Image": item['imageUrl'],
|
||
"DFC:description": description
|
||
}
|
||
endpoint['DFC:supplies'].append(catalogItem)
|
||
|
||
return endpoint
|
||
|
||
|
||
def sharesCatalogEndpoint(baseDir: str, httpPrefix: str,
|
||
domainFull: str,
|
||
path: str) -> {}:
|
||
"""Returns the endpoint for the shares catalog for the instance
|
||
See https://github.com/datafoodconsortium/ontology
|
||
"""
|
||
today, minPrice, maxPrice, matchPattern = _sharesCatalogParams(path)
|
||
dfcUrl = \
|
||
"http://static.datafoodconsortium.org/ontologies/DFC_FullModel.owl#"
|
||
dfcPtUrl = \
|
||
"http://static.datafoodconsortium.org/data/productTypes.rdf#"
|
||
dfcInstanceId = httpPrefix + '://' + domainFull + '/catalog'
|
||
endpoint = {
|
||
"@context": {
|
||
"DFC": dfcUrl,
|
||
"dfc-pt": dfcPtUrl,
|
||
"@base": "http://maPlateformeNationale"
|
||
},
|
||
"@id": dfcInstanceId,
|
||
"@type": "DFC:Entreprise",
|
||
"DFC:supplies": []
|
||
}
|
||
|
||
currDate = datetime.datetime.utcnow()
|
||
currDateStr = currDate.strftime("%Y-%m-%d")
|
||
|
||
for subdir, dirs, files in os.walk(baseDir + '/accounts'):
|
||
for acct in dirs:
|
||
if not isAccountDir(acct):
|
||
continue
|
||
nickname = acct.split('@')[0]
|
||
domain = acct.split('@')[1]
|
||
owner = httpPrefix + '://' + domainFull + '/users/' + nickname
|
||
|
||
sharesFilename = \
|
||
acctDir(baseDir, nickname, domain) + '/shares.json'
|
||
if not os.path.isfile(sharesFilename):
|
||
continue
|
||
print('Test 78363 ' + sharesFilename)
|
||
sharesJson = loadJson(sharesFilename, 1, 2)
|
||
if not sharesJson:
|
||
continue
|
||
|
||
for itemID, item in sharesJson.items():
|
||
if not item.get('dfcId'):
|
||
continue
|
||
if '#' not in item['dfcId']:
|
||
continue
|
||
if today:
|
||
if not item['published'].startswith(currDateStr):
|
||
continue
|
||
if minPrice is not None:
|
||
if float(item['itemPrice']) < minPrice:
|
||
continue
|
||
if maxPrice is not None:
|
||
if float(item['itemPrice']) > maxPrice:
|
||
continue
|
||
description = item['displayName'] + ': ' + item['summary']
|
||
if matchPattern:
|
||
if not re.match(matchPattern, description):
|
||
continue
|
||
|
||
startDateStr = dateSecondsToString(item['published'])
|
||
expireDateStr = dateSecondsToString(item['expire'])
|
||
shareId = _getValidSharedItemID(owner, item['displayName'])
|
||
if item['dfcId'].startswith('epicyon#'):
|
||
dfcId = "epicyon:" + item['dfcId'].split('#')[1]
|
||
else:
|
||
dfcId = "dfc-pt:" + item['dfcId'].split('#')[1]
|
||
priceStr = item['itemPrice'] + ' ' + item['itemCurrency']
|
||
catalogItem = {
|
||
"@id": shareId,
|
||
"@type": "DFC:SuppliedProduct",
|
||
"DFC:hasType": dfcId,
|
||
"DFC:startDate": startDateStr,
|
||
"DFC:expiryDate": expireDateStr,
|
||
"DFC:quantity": float(item['itemQty']),
|
||
"DFC:price": priceStr,
|
||
"DFC:Image": item['imageUrl'],
|
||
"DFC:description": description
|
||
}
|
||
endpoint['DFC:supplies'].append(catalogItem)
|
||
|
||
return endpoint
|
||
|
||
|
||
def sharesCatalogCSVEndpoint(baseDir: str, httpPrefix: str,
|
||
domainFull: str,
|
||
path: str) -> str:
|
||
"""Returns a CSV version of the shares catalog
|
||
"""
|
||
catalogJson = \
|
||
sharesCatalogEndpoint(baseDir, httpPrefix, domainFull, path)
|
||
if not catalogJson:
|
||
return ''
|
||
if not catalogJson.get('DFC:supplies'):
|
||
return ''
|
||
csvStr = \
|
||
'id,type,hasType,startDate,expiryDate,' + \
|
||
'quantity,price,currency,Image,description,\n'
|
||
for item in catalogJson['DFC:supplies']:
|
||
csvStr += '"' + item['@id'] + '",'
|
||
csvStr += '"' + item['@type'] + '",'
|
||
csvStr += '"' + item['DFC:hasType'] + '",'
|
||
csvStr += '"' + item['DFC:startDate'] + '",'
|
||
csvStr += '"' + item['DFC:expiryDate'] + '",'
|
||
csvStr += str(item['DFC:quantity']) + ','
|
||
csvStr += item['DFC:price'].split(' ')[0] + ','
|
||
csvStr += '"' + item['DFC:price'].split(' ')[1] + '",'
|
||
csvStr += '"' + item['DFC:Image'] + '",'
|
||
description = item['DFC:description'].replace('"', "'")
|
||
csvStr += '"' + description + '",\n'
|
||
return csvStr
|
||
|
||
|
||
def generateSharedItemFederationTokens(sharedItemsFederatedDomains: [],
|
||
baseDir: str) -> {}:
|
||
"""Generates tokens for shared item federated domains
|
||
"""
|
||
if not sharedItemsFederatedDomains:
|
||
return {}
|
||
|
||
tokensJson = {}
|
||
if baseDir:
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
if os.path.isfile(tokensFilename):
|
||
tokensJson = loadJson(tokensFilename, 1, 2)
|
||
if tokensJson is None:
|
||
tokensJson = {}
|
||
|
||
tokensAdded = False
|
||
for domainFull in sharedItemsFederatedDomains:
|
||
if not tokensJson.get(domainFull):
|
||
tokensJson[domainFull] = ''
|
||
tokensAdded = True
|
||
|
||
if not tokensAdded:
|
||
return tokensJson
|
||
if baseDir:
|
||
saveJson(tokensJson, tokensFilename)
|
||
return tokensJson
|
||
|
||
|
||
def updateSharedItemFederationToken(baseDir: str,
|
||
tokenDomainFull: str, newToken: str,
|
||
debug: bool,
|
||
tokensJson: {} = None) -> {}:
|
||
"""Updates an individual token for shared item federation
|
||
"""
|
||
if debug:
|
||
print('Updating shared items token for ' + tokenDomainFull)
|
||
if not tokensJson:
|
||
tokensJson = {}
|
||
if baseDir:
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
if os.path.isfile(tokensFilename):
|
||
if debug:
|
||
print('Update loading tokens for ' + tokenDomainFull)
|
||
tokensJson = loadJson(tokensFilename, 1, 2)
|
||
if tokensJson is None:
|
||
tokensJson = {}
|
||
updateRequired = False
|
||
if tokensJson.get(tokenDomainFull):
|
||
if tokensJson[tokenDomainFull] != newToken:
|
||
updateRequired = True
|
||
else:
|
||
updateRequired = True
|
||
if updateRequired:
|
||
tokensJson[tokenDomainFull] = newToken
|
||
if baseDir:
|
||
saveJson(tokensJson, tokensFilename)
|
||
return tokensJson
|
||
|
||
|
||
def mergeSharedItemTokens(baseDir: str, domainFull: str,
|
||
newSharedItemsFederatedDomains: [],
|
||
tokensJson: {}) -> {}:
|
||
"""When the shared item federation domains list has changed, update
|
||
the tokens dict accordingly
|
||
"""
|
||
removals = []
|
||
changed = False
|
||
for tokenDomainFull, tok in tokensJson.items():
|
||
if domainFull:
|
||
if tokenDomainFull.startswith(domainFull):
|
||
continue
|
||
if tokenDomainFull not in newSharedItemsFederatedDomains:
|
||
removals.append(tokenDomainFull)
|
||
# remove domains no longer in the federation list
|
||
for tokenDomainFull in removals:
|
||
del tokensJson[tokenDomainFull]
|
||
changed = True
|
||
# add new domains from the federation list
|
||
for tokenDomainFull in newSharedItemsFederatedDomains:
|
||
if tokenDomainFull not in tokensJson:
|
||
tokensJson[tokenDomainFull] = ''
|
||
changed = True
|
||
if baseDir and changed:
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
saveJson(tokensJson, tokensFilename)
|
||
return tokensJson
|
||
|
||
|
||
def createSharedItemFederationToken(baseDir: str,
|
||
tokenDomainFull: str,
|
||
force: bool,
|
||
tokensJson: {} = None) -> {}:
|
||
"""Updates an individual token for shared item federation
|
||
"""
|
||
if not tokensJson:
|
||
tokensJson = {}
|
||
if baseDir:
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
if os.path.isfile(tokensFilename):
|
||
tokensJson = loadJson(tokensFilename, 1, 2)
|
||
if tokensJson is None:
|
||
tokensJson = {}
|
||
if force or not tokensJson.get(tokenDomainFull):
|
||
tokensJson[tokenDomainFull] = secrets.token_urlsafe(64)
|
||
if baseDir:
|
||
saveJson(tokensJson, tokensFilename)
|
||
return tokensJson
|
||
|
||
|
||
def authorizeSharedItems(sharedItemsFederatedDomains: [],
|
||
baseDir: str,
|
||
originDomainFull: str,
|
||
callingDomainFull: str,
|
||
authHeader: str,
|
||
debug: bool,
|
||
tokensJson: {} = None) -> bool:
|
||
"""HTTP simple token check for shared item federation
|
||
"""
|
||
if not sharedItemsFederatedDomains:
|
||
# no shared item federation
|
||
return False
|
||
if originDomainFull not in sharedItemsFederatedDomains:
|
||
if debug:
|
||
print(originDomainFull +
|
||
' is not in the shared items federation list ' +
|
||
str(sharedItemsFederatedDomains))
|
||
return False
|
||
if 'Basic ' in authHeader:
|
||
if debug:
|
||
print('DEBUG: shared item federation should not use basic auth')
|
||
return False
|
||
providedToken = authHeader.replace('\n', '').replace('\r', '').strip()
|
||
if not providedToken:
|
||
if debug:
|
||
print('DEBUG: shared item federation token is empty')
|
||
return False
|
||
if len(providedToken) < 60:
|
||
if debug:
|
||
print('DEBUG: shared item federation token is too small ' +
|
||
providedToken)
|
||
return False
|
||
if not tokensJson:
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
if not os.path.isfile(tokensFilename):
|
||
if debug:
|
||
print('DEBUG: shared item federation tokens file missing ' +
|
||
tokensFilename)
|
||
return False
|
||
tokensJson = loadJson(tokensFilename, 1, 2)
|
||
if not tokensJson:
|
||
return False
|
||
if not tokensJson.get(callingDomainFull):
|
||
if debug:
|
||
print('DEBUG: shared item federation token ' +
|
||
'check failed for ' + callingDomainFull)
|
||
return False
|
||
if not constantTimeStringCheck(tokensJson[callingDomainFull],
|
||
providedToken):
|
||
if debug:
|
||
print('DEBUG: shared item federation token ' +
|
||
'mismatch for ' + callingDomainFull)
|
||
return False
|
||
return True
|
||
|
||
|
||
def _updateFederatedSharesCache(session, sharedItemsFederatedDomains: [],
|
||
baseDir: str, domainFull: str,
|
||
httpPrefix: str,
|
||
tokensJson: {}, debug: bool,
|
||
systemLanguage: str) -> None:
|
||
"""Updates the cache of federated shares for the instance.
|
||
This enables shared items to be available even when other instances
|
||
might not be online
|
||
"""
|
||
# create directories where catalogs will be stored
|
||
cacheDir = baseDir + '/cache'
|
||
if not os.path.isdir(cacheDir):
|
||
os.mkdir(cacheDir)
|
||
catalogsDir = cacheDir + '/catalogs'
|
||
if not os.path.isdir(catalogsDir):
|
||
os.mkdir(catalogsDir)
|
||
|
||
asHeader = {
|
||
"Accept": "application/ld+json",
|
||
"Origin": domainFull
|
||
}
|
||
for federatedDomainFull in sharedItemsFederatedDomains:
|
||
# NOTE: federatedDomain does not have a port extension,
|
||
# so may not work in some situations
|
||
if federatedDomainFull.startswith(domainFull):
|
||
# only download from instances other than this one
|
||
continue
|
||
if not tokensJson.get(federatedDomainFull):
|
||
# token has been obtained for the other domain
|
||
continue
|
||
if not siteIsActive(httpPrefix + '://' + federatedDomainFull):
|
||
continue
|
||
url = httpPrefix + '://' + federatedDomainFull + '/catalog'
|
||
asHeader['Authorization'] = tokensJson[federatedDomainFull]
|
||
catalogJson = getJson(session, url, asHeader, None,
|
||
debug, __version__, httpPrefix, None)
|
||
if not catalogJson:
|
||
print('WARN: failed to download shared items catalog for ' +
|
||
federatedDomainFull)
|
||
continue
|
||
catalogFilename = catalogsDir + '/' + federatedDomainFull + '.json'
|
||
if saveJson(catalogJson, catalogFilename):
|
||
print('Downloaded shared items catalog for ' + federatedDomainFull)
|
||
sharesJson = _dfcToSharesFormat(catalogJson,
|
||
baseDir, systemLanguage)
|
||
if sharesJson:
|
||
sharesFilename = \
|
||
catalogsDir + '/' + federatedDomainFull + '.shares.json'
|
||
saveJson(sharesJson, sharesFilename)
|
||
print('Converted shares catalog for ' + federatedDomainFull)
|
||
else:
|
||
time.sleep(2)
|
||
|
||
|
||
def runFederatedSharesWatchdog(projectVersion: str, httpd) -> None:
|
||
"""This tries to keep the federated shares update thread
|
||
running even if it dies
|
||
"""
|
||
print('Starting federated shares watchdog')
|
||
federatedSharesOriginal = \
|
||
httpd.thrPostSchedule.clone(runFederatedSharesDaemon)
|
||
httpd.thrFederatedSharesDaemon.start()
|
||
while True:
|
||
time.sleep(55)
|
||
if httpd.thrFederatedSharesDaemon.is_alive():
|
||
continue
|
||
httpd.thrFederatedSharesDaemon.kill()
|
||
httpd.thrFederatedSharesDaemon = \
|
||
federatedSharesOriginal.clone(runFederatedSharesDaemon)
|
||
httpd.thrFederatedSharesDaemon.start()
|
||
print('Restarting federated shares daemon...')
|
||
|
||
|
||
def _generateNextSharesTokenUpdate(baseDir: str,
|
||
minDays: int, maxDays: int) -> None:
|
||
"""Creates a file containing the next date when the shared items token
|
||
for this instance will be updated
|
||
"""
|
||
tokenUpdateFilename = baseDir + '/accounts/.tokenUpdate'
|
||
nextUpdateSec = None
|
||
if os.path.isfile(tokenUpdateFilename):
|
||
with open(tokenUpdateFilename, 'r') as fp:
|
||
nextUpdateStr = fp.read()
|
||
if nextUpdateStr:
|
||
if nextUpdateStr.isdigit():
|
||
nextUpdateSec = int(nextUpdateStr)
|
||
currTime = int(time.time())
|
||
updated = False
|
||
if nextUpdateSec:
|
||
if currTime > nextUpdateSec:
|
||
nextUpdateDays = randint(minDays, maxDays)
|
||
nextUpdateInterval = int(60 * 60 * 24 * nextUpdateDays)
|
||
nextUpdateSec += nextUpdateInterval
|
||
updated = True
|
||
else:
|
||
nextUpdateDays = randint(minDays, maxDays)
|
||
nextUpdateInterval = int(60 * 60 * 24 * nextUpdateDays)
|
||
nextUpdateSec = currTime + nextUpdateInterval
|
||
updated = True
|
||
if updated:
|
||
with open(tokenUpdateFilename, 'w+') as fp:
|
||
fp.write(str(nextUpdateSec))
|
||
|
||
|
||
def _regenerateSharesToken(baseDir: str, domainFull: str,
|
||
minDays: int, maxDays: int, httpd) -> None:
|
||
"""Occasionally the shared items token for your instance is updated.
|
||
Scenario:
|
||
- You share items with $FriendlyInstance
|
||
- Some time later under new management
|
||
$FriendlyInstance becomes $HostileInstance
|
||
- You block $HostileInstance and remove them from your
|
||
federated shares domains list
|
||
- $HostileInstance still knows your shared items token,
|
||
and can still have access to your shared items if it presents a
|
||
spoofed Origin header together with the token
|
||
By rotating the token occasionally $HostileInstance will eventually
|
||
lose access to your federated shares. If other instances within your
|
||
federated shares list of domains continue to follow and communicate
|
||
then they will receive the new token automatically
|
||
"""
|
||
tokenUpdateFilename = baseDir + '/accounts/.tokenUpdate'
|
||
if not os.path.isfile(tokenUpdateFilename):
|
||
return
|
||
nextUpdateSec = None
|
||
with open(tokenUpdateFilename, 'r') as fp:
|
||
nextUpdateStr = fp.read()
|
||
if nextUpdateStr:
|
||
if nextUpdateStr.isdigit():
|
||
nextUpdateSec = int(nextUpdateStr)
|
||
if not nextUpdateSec:
|
||
return
|
||
currTime = int(time.time())
|
||
if currTime <= nextUpdateSec:
|
||
return
|
||
createSharedItemFederationToken(baseDir, domainFull, True, None)
|
||
_generateNextSharesTokenUpdate(baseDir, minDays, maxDays)
|
||
# update the tokens used within the daemon
|
||
httpd.sharedItemFederationTokens = \
|
||
generateSharedItemFederationTokens(httpd.sharedItemsFederatedDomains,
|
||
baseDir)
|
||
|
||
|
||
def runFederatedSharesDaemon(baseDir: str, httpd, httpPrefix: str,
|
||
domainFull: str, proxyType: str, debug: bool,
|
||
systemLanguage: str) -> None:
|
||
"""Runs the daemon used to update federated shared items
|
||
"""
|
||
secondsPerHour = 60 * 60
|
||
fileCheckIntervalSec = 120
|
||
time.sleep(60)
|
||
# the token for this instance will be changed every 7-14 days
|
||
minDays = 7
|
||
maxDays = 14
|
||
_generateNextSharesTokenUpdate(baseDir, minDays, maxDays)
|
||
while True:
|
||
sharedItemsFederatedDomainsStr = \
|
||
getConfigParam(baseDir, 'sharedItemsFederatedDomains')
|
||
if not sharedItemsFederatedDomainsStr:
|
||
time.sleep(fileCheckIntervalSec)
|
||
continue
|
||
|
||
# occasionally change the federated shared items token
|
||
# for this instance
|
||
_regenerateSharesToken(baseDir, domainFull, minDays, maxDays, httpd)
|
||
|
||
# get a list of the domains within the shared items federation
|
||
sharedItemsFederatedDomains = []
|
||
sharedItemsFederatedDomainsList = \
|
||
sharedItemsFederatedDomainsStr.split(',')
|
||
for sharedFederatedDomain in sharedItemsFederatedDomainsList:
|
||
sharedItemsFederatedDomains.append(sharedFederatedDomain.strip())
|
||
if not sharedItemsFederatedDomains:
|
||
time.sleep(fileCheckIntervalSec)
|
||
continue
|
||
|
||
# load the tokens
|
||
tokensFilename = \
|
||
baseDir + '/accounts/sharedItemsFederationTokens.json'
|
||
if not os.path.isfile(tokensFilename):
|
||
time.sleep(fileCheckIntervalSec)
|
||
continue
|
||
tokensJson = loadJson(tokensFilename, 1, 2)
|
||
if not tokensJson:
|
||
time.sleep(fileCheckIntervalSec)
|
||
continue
|
||
|
||
session = createSession(proxyType)
|
||
_updateFederatedSharesCache(session, sharedItemsFederatedDomains,
|
||
baseDir, domainFull, httpPrefix,
|
||
tokensJson, debug, systemLanguage)
|
||
time.sleep(secondsPerHour * 6)
|
||
|
||
|
||
def _dfcToSharesFormat(catalogJson: {},
|
||
baseDir: str, systemLanguage: str) -> {}:
|
||
"""Converts DFC format into the internal formal used to store shared items.
|
||
This simplifies subsequent search and display
|
||
"""
|
||
if not catalogJson.get('DFC:supplies'):
|
||
return {}
|
||
sharesJson = {}
|
||
|
||
dfcIds = {}
|
||
productTypesList = getCategoryTypes(baseDir)
|
||
for productType in productTypesList:
|
||
dfcIds[productType] = _loadDfcIds(baseDir, systemLanguage, productType)
|
||
|
||
currTime = int(time.time())
|
||
for item in catalogJson['DFC:supplies']:
|
||
if not item.get('@id') or \
|
||
not item.get('@type') or \
|
||
not item.get('DFC:hasType') or \
|
||
not item.get('DFC:startDate') or \
|
||
not item.get('DFC:expiryDate') or \
|
||
not item.get('DFC:quantity') or \
|
||
not item.get('DFC:price') or \
|
||
not item.get('DFC:Image') or \
|
||
not item.get('DFC:description'):
|
||
continue
|
||
|
||
if ' ' not in item['DFC:price']:
|
||
continue
|
||
if ':' not in item['DFC:description']:
|
||
continue
|
||
if ':' not in item['DFC:hasType']:
|
||
continue
|
||
|
||
startTimeSec = dateStringToSeconds(item['DFC:startDate'])
|
||
if not startTimeSec:
|
||
continue
|
||
expiryTimeSec = dateStringToSeconds(item['DFC:expiryDate'])
|
||
if not expiryTimeSec:
|
||
continue
|
||
if expiryTimeSec < currTime:
|
||
# has expired
|
||
continue
|
||
|
||
if item['DFC:hasType'].startswith('epicyon:'):
|
||
itemType = item['DFC:hasType'].split(':')[1]
|
||
itemType = itemType.replace('_', ' ')
|
||
itemCategory = 'non-food'
|
||
productType = None
|
||
else:
|
||
hasType = item['DFC:hasType'].split(':')[1]
|
||
itemType = None
|
||
productType = None
|
||
for prodType in productTypesList:
|
||
itemType = _getshareTypeFromDfcId(hasType, dfcIds[prodType])
|
||
if itemType:
|
||
productType = prodType
|
||
break
|
||
itemCategory = 'food'
|
||
if not itemType:
|
||
continue
|
||
|
||
allText = item['DFC:description'] + ' ' + itemType + ' ' + itemCategory
|
||
if isFilteredGlobally(baseDir, allText):
|
||
continue
|
||
|
||
dfcId = None
|
||
if productType:
|
||
dfcId = dfcIds[productType][itemType]
|
||
itemID = item['@id']
|
||
description = item['DFC:description'].split(':', 1)[1].strip()
|
||
|
||
sharesJson[itemID] = {
|
||
"displayName": item['DFC:description'].split(':')[0],
|
||
"summary": description,
|
||
"imageUrl": item['DFC:Image'],
|
||
"itemQty": float(item['DFC:quantity']),
|
||
"dfcId": dfcId,
|
||
"itemType": itemType,
|
||
"category": itemCategory,
|
||
"location": "",
|
||
"published": startTimeSec,
|
||
"expire": expiryTimeSec,
|
||
"itemPrice": item['DFC:price'].split(' ')[0],
|
||
"itemCurrency": item['DFC:price'].split(' ')[1]
|
||
}
|
||
return sharesJson
|