| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | __filename__ = "metadata.py" | 
					
						
							|  |  |  | __author__ = "Bob Mottram" | 
					
						
							|  |  |  | __license__ = "AGPL3+" | 
					
						
							| 
									
										
										
										
											2022-02-03 13:58:20 +00:00
										 |  |  | __version__ = "1.3.0" | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | __maintainer__ = "Bob Mottram" | 
					
						
							| 
									
										
										
										
											2021-09-10 16:14:50 +00:00
										 |  |  | __email__ = "bob@libreserver.org" | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | __status__ = "Production" | 
					
						
							| 
									
										
										
										
											2021-06-15 15:08:12 +00:00
										 |  |  | __module_group__ = "Metadata" | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  | import os | 
					
						
							| 
									
										
										
										
											2021-12-26 18:46:43 +00:00
										 |  |  | from utils import is_account_dir | 
					
						
							| 
									
										
										
										
											2021-12-26 15:13:34 +00:00
										 |  |  | from utils import load_json | 
					
						
							| 
									
										
										
										
											2021-12-28 14:41:10 +00:00
										 |  |  | from utils import no_of_accounts | 
					
						
							|  |  |  | from utils import no_of_active_accounts_monthly | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-12-29 21:55:09 +00:00
										 |  |  | def _get_status_count(base_dir: str) -> int: | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |     """Get the total number of posts
 | 
					
						
							|  |  |  |     """
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     status_ctr = 0 | 
					
						
							|  |  |  |     accounts_dir = base_dir + '/accounts' | 
					
						
							|  |  |  |     for _, dirs, _ in os.walk(accounts_dir): | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |         for acct in dirs: | 
					
						
							| 
									
										
										
										
											2021-12-26 18:46:43 +00:00
										 |  |  |             if not is_account_dir(acct): | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |                 continue | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             acct_dir = os.path.join(accounts_dir, acct + '/outbox') | 
					
						
							|  |  |  |             for _, _, files2 in os.walk(acct_dir): | 
					
						
							|  |  |  |                 status_ctr += len(files2) | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |                 break | 
					
						
							|  |  |  |         break | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     return status_ctr | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-12-28 17:20:43 +00:00
										 |  |  | def meta_data_node_info(base_dir: str, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                         about_url: str, | 
					
						
							|  |  |  |                         terms_of_service_url: str, | 
					
						
							| 
									
										
										
										
											2021-12-28 17:20:43 +00:00
										 |  |  |                         registration: bool, version: str, | 
					
						
							|  |  |  |                         showAccounts: bool) -> {}: | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |     """ /nodeinfo/2.0 endpoint
 | 
					
						
							| 
									
										
										
										
											2021-05-03 10:05:05 +00:00
										 |  |  |     Also see https://socialhub.activitypub.rocks/t/ | 
					
						
							|  |  |  |     fep-f1d5-nodeinfo-in-fediverse-software/1190/4 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     Note that there are security considerations with this. If an adversary | 
					
						
							|  |  |  |     sees a lot of accounts and "local" posts then the instance may be | 
					
						
							|  |  |  |     considered a higher priority target. | 
					
						
							|  |  |  |     Also exposure of the version number and number of accounts could be | 
					
						
							|  |  |  |     sensitive | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |     """
 | 
					
						
							| 
									
										
										
										
											2021-05-03 10:05:05 +00:00
										 |  |  |     if showAccounts: | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         active_accounts = no_of_accounts(base_dir) | 
					
						
							|  |  |  |         active_accounts_monthly = no_of_active_accounts_monthly(base_dir, 1) | 
					
						
							|  |  |  |         active_accounts_half_year = no_of_active_accounts_monthly(base_dir, 6) | 
					
						
							|  |  |  |         local_posts = _get_status_count(base_dir) | 
					
						
							| 
									
										
										
										
											2021-05-03 10:05:05 +00:00
										 |  |  |     else: | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         active_accounts = 1 | 
					
						
							|  |  |  |         active_accounts_monthly = 1 | 
					
						
							|  |  |  |         active_accounts_half_year = 1 | 
					
						
							|  |  |  |         local_posts = 1 | 
					
						
							| 
									
										
										
										
											2021-05-03 10:05:05 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  |     nodeinfo = { | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |         'openRegistrations': registration, | 
					
						
							|  |  |  |         'protocols': ['activitypub'], | 
					
						
							|  |  |  |         'software': { | 
					
						
							|  |  |  |             'name': 'epicyon', | 
					
						
							|  |  |  |             'version': version | 
					
						
							|  |  |  |         }, | 
					
						
							| 
									
										
										
										
											2021-05-04 09:39:15 +00:00
										 |  |  |         'documents': { | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'about': about_url, | 
					
						
							|  |  |  |             'terms': terms_of_service_url | 
					
						
							| 
									
										
										
										
											2021-05-04 09:39:15 +00:00
										 |  |  |         }, | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |         'usage': { | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'localPosts': local_posts, | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |             'users': { | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                 'activeHalfyear': active_accounts_half_year, | 
					
						
							|  |  |  |                 'activeMonth': active_accounts_monthly, | 
					
						
							|  |  |  |                 'total': active_accounts | 
					
						
							| 
									
										
										
										
											2019-11-13 10:32:12 +00:00
										 |  |  |             } | 
					
						
							|  |  |  |         }, | 
					
						
							|  |  |  |         'version': '2.0' | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  |     return nodeinfo | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-12-29 21:55:09 +00:00
										 |  |  | def meta_data_instance(showAccounts: bool, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                        instance_title: str, | 
					
						
							|  |  |  |                        instance_description_short: str, | 
					
						
							|  |  |  |                        instance_description: str, | 
					
						
							| 
									
										
										
										
											2021-12-29 21:55:09 +00:00
										 |  |  |                        http_prefix: str, base_dir: str, | 
					
						
							| 
									
										
										
										
											2021-12-31 21:18:12 +00:00
										 |  |  |                        admin_nickname: str, domain: str, domain_full: str, | 
					
						
							| 
									
										
										
										
											2021-12-29 21:55:09 +00:00
										 |  |  |                        registration: bool, system_language: str, | 
					
						
							|  |  |  |                        version: str) -> {}: | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |     """ /api/v1/instance endpoint
 | 
					
						
							|  |  |  |     """
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     admin_actor_filename = \ | 
					
						
							| 
									
										
										
										
											2021-12-31 21:18:12 +00:00
										 |  |  |         base_dir + '/accounts/' + admin_nickname + '@' + domain + '.json' | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     if not os.path.isfile(admin_actor_filename): | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         return {} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     admin_actor = load_json(admin_actor_filename, 0) | 
					
						
							|  |  |  |     if not admin_actor: | 
					
						
							| 
									
										
										
										
											2021-12-29 21:55:09 +00:00
										 |  |  |         print('WARN: json load exception meta_data_instance') | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         return {} | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     rules_list = [] | 
					
						
							|  |  |  |     rules_filename = \ | 
					
						
							| 
									
										
										
										
											2021-12-25 16:17:53 +00:00
										 |  |  |         base_dir + '/accounts/tos.md' | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     if os.path.isfile(rules_filename): | 
					
						
							|  |  |  |         with open(rules_filename, 'r') as fp_rules: | 
					
						
							|  |  |  |             rules_lines = fp_rules.readlines() | 
					
						
							|  |  |  |             rule_ctr = 1 | 
					
						
							|  |  |  |             for line in rules_lines: | 
					
						
							| 
									
										
										
										
											2021-10-12 11:09:04 +00:00
										 |  |  |                 line = line.strip() | 
					
						
							|  |  |  |                 if not line: | 
					
						
							|  |  |  |                     continue | 
					
						
							|  |  |  |                 if line.startswith('#'): | 
					
						
							|  |  |  |                     continue | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                 rules_list.append({ | 
					
						
							|  |  |  |                     'id': str(rule_ctr), | 
					
						
							| 
									
										
										
										
											2021-10-12 11:09:04 +00:00
										 |  |  |                     'text': line | 
					
						
							|  |  |  |                 }) | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                 rule_ctr += 1 | 
					
						
							| 
									
										
										
										
											2021-10-12 11:09:04 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     is_bot = False | 
					
						
							|  |  |  |     is_group = False | 
					
						
							|  |  |  |     if admin_actor['type'] == 'Group': | 
					
						
							|  |  |  |         is_group = True | 
					
						
							|  |  |  |     elif admin_actor['type'] != 'Person': | 
					
						
							|  |  |  |         is_bot = True | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  | 
 | 
					
						
							|  |  |  |     url = \ | 
					
						
							| 
									
										
										
										
											2021-12-26 10:00:46 +00:00
										 |  |  |         http_prefix + '://' + domain_full + '/@' + \ | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         admin_actor['preferredUsername'] | 
					
						
							| 
									
										
										
										
											2020-03-22 21:16:02 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |     if showAccounts: | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         active_accounts = no_of_accounts(base_dir) | 
					
						
							|  |  |  |         local_posts = _get_status_count(base_dir) | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  |     else: | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         active_accounts = 1 | 
					
						
							|  |  |  |         local_posts = 1 | 
					
						
							| 
									
										
										
										
											2021-06-05 09:22:35 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     created_at = '' | 
					
						
							|  |  |  |     if admin_actor.get('published'): | 
					
						
							|  |  |  |         created_at = admin_actor['published'] | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  |     instance = { | 
					
						
							| 
									
										
										
										
											2019-11-13 12:55:37 +00:00
										 |  |  |         'approval_required': False, | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |         'invites_enabled': False, | 
					
						
							|  |  |  |         'registrations': registration, | 
					
						
							| 
									
										
										
										
											2020-03-22 20:36:19 +00:00
										 |  |  |         'contact_account': { | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'acct': admin_actor['preferredUsername'], | 
					
						
							|  |  |  |             'created_at': created_at, | 
					
						
							|  |  |  |             'avatar': admin_actor['icon']['url'], | 
					
						
							|  |  |  |             'avatar_static': admin_actor['icon']['url'], | 
					
						
							|  |  |  |             'header': admin_actor['image']['url'], | 
					
						
							|  |  |  |             'header_static': admin_actor['image']['url'], | 
					
						
							|  |  |  |             'bot': is_bot, | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |             'discoverable': True, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'group': is_group, | 
					
						
							|  |  |  |             'display_name': admin_actor['name'], | 
					
						
							|  |  |  |             'locked': admin_actor['manuallyApprovesFollowers'], | 
					
						
							| 
									
										
										
										
											2021-05-04 10:17:06 +00:00
										 |  |  |             'note': '<p>Admin of ' + domain + '</p>', | 
					
						
							| 
									
										
										
										
											2020-04-03 16:59:12 +00:00
										 |  |  |             'url': url, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'username': admin_actor['preferredUsername'] | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         }, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         'description': instance_description, | 
					
						
							| 
									
										
										
										
											2021-12-25 23:03:28 +00:00
										 |  |  |         'languages': [system_language], | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         'short_description': instance_description_short, | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         'stats': { | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |             'domain_count': 2, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             'status_count': local_posts, | 
					
						
							|  |  |  |             'user_count': active_accounts | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         }, | 
					
						
							| 
									
										
										
										
											2021-12-26 10:00:46 +00:00
										 |  |  |         'thumbnail': http_prefix + '://' + domain_full + '/login.png', | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         'title': instance_title, | 
					
						
							| 
									
										
										
										
											2021-12-26 10:00:46 +00:00
										 |  |  |         'uri': domain_full, | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |         'urls': {}, | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |         'version': version, | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |         'rules': rules_list, | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |         'configuration': { | 
					
						
							|  |  |  |             'statuses': { | 
					
						
							|  |  |  |                 'max_media_attachments': 1 | 
					
						
							|  |  |  |             }, | 
					
						
							|  |  |  |             'media_attachments': { | 
					
						
							|  |  |  |                 'supported_mime_types': [ | 
					
						
							|  |  |  |                     'image/jpeg', | 
					
						
							| 
									
										
										
										
											2022-02-06 11:04:49 +00:00
										 |  |  |                     'image/jxl', | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |                     'image/png', | 
					
						
							|  |  |  |                     'image/gif', | 
					
						
							|  |  |  |                     'image/webp', | 
					
						
							|  |  |  |                     'image/avif', | 
					
						
							|  |  |  |                     'image/svg+xml', | 
					
						
							|  |  |  |                     'video/webm', | 
					
						
							|  |  |  |                     'video/mp4', | 
					
						
							|  |  |  |                     'video/ogv', | 
					
						
							|  |  |  |                     'audio/ogg', | 
					
						
							| 
									
										
										
										
											2022-04-18 13:21:45 +00:00
										 |  |  |                     'audio/opus', | 
					
						
							| 
									
										
										
										
											2021-10-12 10:31:22 +00:00
										 |  |  |                     'audio/flac', | 
					
						
							|  |  |  |                     'audio/mpeg' | 
					
						
							|  |  |  |                 ], | 
					
						
							|  |  |  |                 'image_size_limit': 10485760, | 
					
						
							|  |  |  |                 'image_matrix_limit': 16777216, | 
					
						
							|  |  |  |                 'video_size_limit': 41943040, | 
					
						
							|  |  |  |                 'video_frame_rate_limit': 60, | 
					
						
							|  |  |  |                 'video_matrix_limit': 2304000 | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  |         } | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |     } | 
					
						
							| 
									
										
										
										
											2020-03-22 21:16:02 +00:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-11-13 12:45:41 +00:00
										 |  |  |     return instance | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-12-28 17:20:43 +00:00
										 |  |  | def metadata_custom_emoji(base_dir: str, | 
					
						
							|  |  |  |                           http_prefix: str, domain_full: str) -> {}: | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  |     """Returns the custom emoji
 | 
					
						
							|  |  |  |     Endpoint /api/v1/custom_emojis | 
					
						
							|  |  |  |     See https://docs.joinmastodon.org/methods/instance/custom_emojis | 
					
						
							|  |  |  |     """
 | 
					
						
							|  |  |  |     result = [] | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |     emojis_url = http_prefix + '://' + domain_full + '/emoji' | 
					
						
							|  |  |  |     for _, _, files in os.walk(base_dir + '/emoji'): | 
					
						
							|  |  |  |         for fname in files: | 
					
						
							|  |  |  |             if len(fname) < 3: | 
					
						
							| 
									
										
										
										
											2021-05-27 22:19:52 +00:00
										 |  |  |                 continue | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             if fname[0].isdigit() or fname[1].isdigit(): | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  |                 continue | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             if not fname.endswith('.png'): | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  |                 continue | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |             url = os.path.join(emojis_url, fname) | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  |             result.append({ | 
					
						
							| 
									
										
										
										
											2022-01-03 10:27:55 +00:00
										 |  |  |                 "shortcode": fname.replace('.png', ''), | 
					
						
							| 
									
										
										
										
											2021-05-27 22:08:49 +00:00
										 |  |  |                 "url": url, | 
					
						
							|  |  |  |                 "static_url": url, | 
					
						
							|  |  |  |                 "visible_in_picker": True | 
					
						
							|  |  |  |             }) | 
					
						
							|  |  |  |         break | 
					
						
							|  |  |  |     return result |