From 5ed67744a213cb4c586d7f9dcee818cc1dc5c853 Mon Sep 17 00:00:00 2001 From: Benjamin 'blindCoder' Schieder Date: Sun, 23 Apr 2017 20:49:07 +0200 Subject: [PATCH] final touches --- Tweetodon/DB.pm | 14 +--- Tweetodon/Feed.pm | 19 ++++- Tweetodon/Migration.pm | 17 ++++ Tweetodon/Website/EditFeed.pm | 1 + cronjob.pl | 79 +++++++++++++++++++ migrations/0000-dbinit.sql | 139 ++++++++++++++++++++------------- post_status.bash | 4 + static/templates/EditFeed.html | 12 ++- update_db.pl | 36 +++++++++ 9 files changed, 253 insertions(+), 68 deletions(-) create mode 100644 Tweetodon/Migration.pm create mode 100755 cronjob.pl create mode 100755 post_status.bash create mode 100755 update_db.pl diff --git a/Tweetodon/DB.pm b/Tweetodon/DB.pm index f6e7eea..6b6d54f 100644 --- a/Tweetodon/DB.pm +++ b/Tweetodon/DB.pm @@ -150,18 +150,10 @@ sub doUPDATEhash { sub DBInit { unless (defined($DBH)){ - my $config = ""; - open CONFIG, "tweetodon.conf.json" or die "Cannot open ttiam.conf.json"; - { - $/ = undef; - $config = ; - } - close CONFIG; - $config = decode_json($config); - my $DSN = "DBI:mysql:database=".$config->{"mysql"}->{"database"}. - ":host=".$config->{"mysql"}->{"host"}. + my $DSN = "DBI:mysql:database=".$main::config->{"mysql"}->{"database"}. + ":host=".$main::config->{"mysql"}->{"host"}. ";mysql_emulated_prepare=1"; - $DBH = DBI->connect($DSN, $config->{"mysql"}->{"user"}, $config->{"mysql"}->{"pass"}); + $DBH = DBI->connect($DSN, $main::config->{"mysql"}->{"user"}, $main::config->{"mysql"}->{"pass"}); unless ($DBH){ main::Error("SQL Error", "Database Error", "Unable to connect to database '$DSN'!\nError: $DBI::errstr\n"); } diff --git a/Tweetodon/Feed.pm b/Tweetodon/Feed.pm index 93e19bf..e84de85 100644 --- a/Tweetodon/Feed.pm +++ b/Tweetodon/Feed.pm @@ -54,12 +54,29 @@ sub filters { } return @retVal; } - sub fetch_entries { my $self = shift; $XML::Feed::MULTIPLE_ENCLOSURES = 1; return XML::Feed->parse(URI->new($self->{data}->{url})); } +sub entry_by { + my $self = shift; + my $key = shift; + my $value = shift; + + my $sth = Tweetodon::DB->doSELECT("SELECT * FROM entries WHERE feed_id = ? AND $key = ?", $self->{data}->{ID}, $value); + my @retVal; + foreach my $r (@$sth){ + push @retVal, Tweetodon::Entry->new($r); + } + return @retVal; +} +sub user { + my $self = shift; + my $sth = Tweetodon::DB->doSELECT("SELECT * FROM users WHERE username = ? and instance = ?", $self->{data}->{username}, $self->{data}->{instance}); + $sth = $$sth[0]; + return Tweetodon::User->new($sth); +} 1; diff --git a/Tweetodon/Migration.pm b/Tweetodon/Migration.pm new file mode 100644 index 0000000..a42de7e --- /dev/null +++ b/Tweetodon/Migration.pm @@ -0,0 +1,17 @@ +# vim: set foldmarker={,}: +use strict; +use Tweetodon::Base; + +package Tweetodon::Migration; +@Tweetodon::Migration::ISA = qw(Tweetodon::Base); +use JSON; +use Data::Dumper; + +sub dbTable :lvalue { "tokens"; } +sub orderBy :lvalue { "username ASC"; } + +# Class functions + +# Object methods + +1; diff --git a/Tweetodon/Website/EditFeed.pm b/Tweetodon/Website/EditFeed.pm index 453d75e..d404c1e 100644 --- a/Tweetodon/Website/EditFeed.pm +++ b/Tweetodon/Website/EditFeed.pm @@ -106,6 +106,7 @@ sub fill_content { $output->param("url", $feed->{data}->{url}); $output->param("feed_id", $feed->{data}->{ID}); + $output->param("format", $feed->{data}->{format}); return 1; } sub prerender { diff --git a/cronjob.pl b/cronjob.pl new file mode 100755 index 0000000..2d7816d --- /dev/null +++ b/cronjob.pl @@ -0,0 +1,79 @@ +#!/usr/bin/perl -w -I. + +use strict; +use Data::Dumper; +use URI::Escape; +use JSON; +use Tweetodon::Feed; +use Tweetodon::User; +use Tweetodon::Entry; + +our $config = ""; +open CONFIG, "tweetodon.conf.json" or die "Cannot open tweetodon.conf.json"; +{ + $/ = undef; + $config = ; +} +close CONFIG; +$config = decode_json($config); + +sub Error {{{ + my $errormessage = "\nStack Trace:\n"; + + my $i=0; + while ((my @call_details = (caller($i++))) ){ + $errormessage .= $call_details[1].":".$call_details[2]." in function ".$call_details[3]."\n"; + } + + print STDERR $errormessage; + exit(1); +}}} + +# Force Unicode output +binmode STDERR, ":utf8"; +binmode STDOUT, ":utf8"; + +my @feeds = Tweetodon::Feed->all(); +FEED: foreach my $feed (@feeds){ + next FEED unless $feed->{data}->{enabled}; + my $entries = $feed->fetch_entries(); + ENTRY: foreach my $entry ($entries->items){ + my @seen_entries = $feed->entry_by("entry_link", $entry->link()); + next ENTRY if ((scalar @seen_entries) > 0); + + my %entry; + $entry{title} = $entry->title(); + $entry{link} = $entry->link(); + $entry{content} = $entry->content()->body; + $entry{author} = $entry->author(); + $entry{issued} = $entry->issued(); + $entry{modified} = $entry->modified(); + $entry{id} = $entry->id(); + $entry{tags} = join(", ", $entry->tags()); + + my $user = $feed->user(); + my $status = $feed->{data}->{format}; + $status =~ s/{ID}/$entry{id}/g; + $status =~ s/{Title}/$entry{title}/g; + $status =~ s/{Link}/$entry{link}/g; + $status =~ s/{Content}/$entry{content}/g; + $status =~ s/{Author}/$entry{author}/g; + $status =~ s/{Issued}/$entry{issued}/g; + $status =~ s/{Modified}/$entry{modified}/g; + $status =~ s/{Tags}/$entry{tags}/g; + + $status =~ s/'/"/g; # TODO + + open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' '$status'|"); + my $reply = ""; + { + $/ = undef; + $reply = ; + } + + my %ne; + $ne{feed_id} = $feed->{data}->{ID}; + $ne{entry_link} = $entry{link}; + Tweetodon::Entry->create(%ne); + } +} diff --git a/migrations/0000-dbinit.sql b/migrations/0000-dbinit.sql index 41c5507..e85a538 100644 --- a/migrations/0000-dbinit.sql +++ b/migrations/0000-dbinit.sql @@ -1,67 +1,96 @@ --- phpMyAdmin SQL Dump --- version 4.4.0 --- http://www.phpmyadmin.net --- --- Host: mysql.pallas.crash-override.net --- Erstellungszeit: 12. Okt 2015 um 13:23 --- Server-Version: 5.6.20 --- PHP-Version: 5.6.9-1 - SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +SET AUTOCOMMIT = 0; +START TRANSACTION; SET time_zone = "+00:00"; --- --- Datenbank: `ttiam_development` --- +CREATE TABLE `apps` ( + `ID` bigint(20) UNSIGNED NOT NULL, + `instance` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `instance_id` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `instance_client_id` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `instance_client_secret` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; --- -------------------------------------------------------- +CREATE TABLE `entries` ( + `ID` bigint(20) UNSIGNED NOT NULL, + `feed_id` bigint(20) UNSIGNED NOT NULL, + `entry_link` text NOT NULL, + `posted_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB DEFAULT CHARSET=latin1; --- --- Tabellenstruktur für Tabelle `accounts` --- +CREATE TABLE `feeds` ( + `ID` bigint(20) UNSIGNED NOT NULL, + `username` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `instance` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `url` text CHARACTER SET utf8mb4 COLLATE utf8_unicode_ci NOT NULL, + `format` varchar(500) NOT NULL DEFAULT '{Title} - {Link} by {Author} -- posted at {Issued} with #Tweetodon', + `enabled` enum('0','1') NOT NULL DEFAULT '0' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; -DROP TABLE IF EXISTS `accounts`; -CREATE TABLE IF NOT EXISTS `accounts` ( - `id` bigint(20) unsigned NOT NULL, - `application_id` bigint(20) unsigned NOT NULL, - `identity_id` bigint(20) unsigned NOT NULL, +CREATE TABLE `filters` ( + `ID` bigint(20) UNSIGNED NOT NULL, + `feed_id` bigint(20) UNSIGNED NOT NULL, + `field` text NOT NULL, + `regex` text NOT NULL, + `type` enum('white','black') NOT NULL, + `match` enum('positive','negative') NOT NULL DEFAULT 'positive' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE `migrations` ( + `ID` bigint(20) UNSIGNED NOT NULL, + `name` text NOT NULL, + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE `users` ( + `ID` bigint(20) NOT NULL, `username` text NOT NULL, - `sha1_username` varchar(40) NOT NULL, - `configuration` text NOT NULL, - `valid_from` datetime DEFAULT NULL, - `valid_until` datetime DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; + `username_sha256` varchar(128) NOT NULL, + `instance` text NOT NULL, + `instance_sha256` varchar(128) NOT NULL, + `access_token` text NOT NULL, + `session_id` text NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; --- --- Indizes der exportierten Tabellen --- --- --- Indizes für die Tabelle `accounts` --- -ALTER TABLE `accounts` - ADD PRIMARY KEY (`id`), - ADD UNIQUE KEY `sha1_username` (`sha1_username`), - ADD UNIQUE KEY `application_username` (`application_id`,`sha1_username`), - ADD KEY `identity_id` (`identity_id`), - ADD KEY `application_id` (`application_id`); +ALTER TABLE `apps` + ADD PRIMARY KEY (`ID`); --- --- AUTO_INCREMENT für exportierte Tabellen --- +ALTER TABLE `entries` + ADD PRIMARY KEY (`ID`), + ADD KEY `feed_id` (`feed_id`); --- --- AUTO_INCREMENT für Tabelle `accounts` --- -ALTER TABLE `accounts` - MODIFY `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT; --- --- Constraints der exportierten Tabellen --- +ALTER TABLE `feeds` + ADD PRIMARY KEY (`ID`); --- --- Constraints der Tabelle `accounts` --- -ALTER TABLE `accounts` - ADD CONSTRAINT `accounts_ibfk_1` FOREIGN KEY (`application_id`) REFERENCES `applications` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - ADD CONSTRAINT `accounts_ibfk_2` FOREIGN KEY (`identity_id`) REFERENCES `identities` (`id`) ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE `filters` + ADD PRIMARY KEY (`ID`), + ADD KEY `feed_id` (`feed_id`); + +ALTER TABLE `migrations` + ADD PRIMARY KEY (`ID`); + +ALTER TABLE `users` + ADD PRIMARY KEY (`ID`), + ADD UNIQUE KEY `user` (`username_sha256`,`instance_sha256`); + + +ALTER TABLE `apps` + MODIFY `ID` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT; +ALTER TABLE `entries` + MODIFY `ID` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT; +ALTER TABLE `feeds` + MODIFY `ID` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT; +ALTER TABLE `filters` + MODIFY `ID` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT; +ALTER TABLE `migrations` + MODIFY `ID` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT; +ALTER TABLE `users` + MODIFY `ID` bigint(20) NOT NULL AUTO_INCREMENT; + +ALTER TABLE `entries` + ADD CONSTRAINT `feed_id` FOREIGN KEY (`feed_id`) REFERENCES `feeds` (`ID`) ON DELETE CASCADE ON UPDATE CASCADE; + +ALTER TABLE `filters` + ADD CONSTRAINT `filters_ibfk_1` FOREIGN KEY (`feed_id`) REFERENCES `feeds` (`ID`) ON DELETE CASCADE ON UPDATE CASCADE; +COMMIT; diff --git a/post_status.bash b/post_status.bash new file mode 100755 index 0000000..ba192d8 --- /dev/null +++ b/post_status.bash @@ -0,0 +1,4 @@ +#!/bin/bash + +curl -sS -X POST --header "Authorization: Bearer ${1}" "${2}/api/v1/statuses" -d "status=${3}" 2>/dev/null +#https://cloud.digitalocean.com/v1/oauth/token?client_id=CLIENT_ID&client_secret=CLIENT_SECRET&grant_type=authorization_code&code=AUTHORIZATION_CODE&redirect_uri=CALLBACK_URL diff --git a/static/templates/EditFeed.html b/static/templates/EditFeed.html index 939fd9e..a588900 100644 --- a/static/templates/EditFeed.html +++ b/static/templates/EditFeed.html @@ -39,6 +39,16 @@ +
+ +

Format

+
+ Enter the format for the toot. You can use the placeholders {ID}, {Title}, {Link}, {Content}, {Author}, {Issued}, {Modified} and {Tags} to substitute the respective values.
+ " class="btn-block">
+ +
+
+

Filters

@@ -56,7 +66,7 @@ - Save filters Add filter + Save filters Add filter
diff --git a/update_db.pl b/update_db.pl new file mode 100755 index 0000000..be8bd68 --- /dev/null +++ b/update_db.pl @@ -0,0 +1,36 @@ +#!/usr/bin/perl -w -I. + +use strict; +use Data::Dumper; +use Tweetodon::Migration; + +our $config = ""; +open CONFIG, "tweetodon.conf.json" or die "Cannot open tweetodon.conf.json"; +{ + $/ = undef; + $config = ; +} +close CONFIG; +$config = decode_json($config); + +# Force Unicode output +binmode STDERR, ":utf8"; +binmode STDOUT, ":utf8"; + +my @migrations = glob ("migrations/*sql"); +foreach my $migration (@migrations){ + print "Running migration $migration\n"; + if (!Tweetodon::Migration->get_by("name", $migration)){ + open (M, $migration); + my $sql; + { + $/ = undef; + $sql = ; + } + close M; + Tweetodon::DB->doINSERT($sql); + my %migdata; + $migdata{name} = $migration; + Tweetodon::Migration->create(%migdata); + } +}