From 22962f88c5936a4c0c278117e1cdec69e4a17298 Mon Sep 17 00:00:00 2001 From: mj-saunders Date: Mon, 21 May 2018 14:08:33 +0000 Subject: [PATCH 1/3] Prepare db for toot digests --- migrations/0002-allow_toot_digests.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 migrations/0002-allow_toot_digests.sql diff --git a/migrations/0002-allow_toot_digests.sql b/migrations/0002-allow_toot_digests.sql new file mode 100644 index 0000000..d806050 --- /dev/null +++ b/migrations/0002-allow_toot_digests.sql @@ -0,0 +1,5 @@ +ALTER TABLE `feeds` + ADD `digest_enabled` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 AFTER `enabled`, + ADD `digest_limit` tinyint UNSIGNED NOT NULL DEFAULT 5 AFTER `digest_enabled`, + ADD `digest_signature` varchar(100) AFTER `digest_limit`; + From 9b84f3054741a90b8ed53a863e485b2b543e9806 Mon Sep 17 00:00:00 2001 From: mj-saunders Date: Sat, 26 May 2018 10:31:35 +0000 Subject: [PATCH 2/3] Update front end to display and edit toot digest settings --- RSSTootalizer/Website/EditFeed.pm | 18 ++++++++++++++++++ static/css/EditFeed.css | 5 +++++ static/js/EditFeed.js | 11 +++++++++++ static/templates/EditFeed.html | 13 +++++++++++++ 4 files changed, 47 insertions(+) diff --git a/RSSTootalizer/Website/EditFeed.pm b/RSSTootalizer/Website/EditFeed.pm index 482f4b1..47b882e 100644 --- a/RSSTootalizer/Website/EditFeed.pm +++ b/RSSTootalizer/Website/EditFeed.pm @@ -63,6 +63,20 @@ sub fill_content { } } + if ($main::FORM{enabledigest} and "x".$main::FORM{enabledigest} eq "x".$feed->{data}->{ID}){ + $feed->{data}->{digest_enabled} = 1; + $feed->save(); + } + if ($main::FORM{disabledigest} and "x".$main::FORM{disabledigest} eq "x".$feed->{data}->{ID}){ + $feed->{data}->{digest_enabled} = 0; + $feed->save(); + } + if ($main::FORM{action} and "x".$main::FORM{action} eq "xsavedigest"){ + $feed->{data}->{digest_limit} = $main::FORM{digestlimit}; + $feed->{data}->{digest_signature} = $main::FORM{digestsig}; + $feed->save(); + } + my @param_entries; my @filters = $feed->filters(); my $feeddata = $feed->fetch_entries(); @@ -106,6 +120,10 @@ sub fill_content { $output->param("url", $feed->{data}->{url}); $output->param("feed_id", $feed->{data}->{ID}); $output->param("format", $feed->{data}->{format}); + + $output->param("digestenabled", $feed->{data}->{digest_enabled}); + $output->param("digestlimit", $feed->{data}->{digest_limit}); + $output->param("digestsig", $feed->{data}->{digest_signature}); return 1; } sub prerender { diff --git a/static/css/EditFeed.css b/static/css/EditFeed.css index c970c70..2be0ab4 100644 --- a/static/css/EditFeed.css +++ b/static/css/EditFeed.css @@ -4,3 +4,8 @@ .red { background-color: #FF8080; } + +#diglim { + font-family: Consolas, Lucida Console, monospace; + width: 3em; +} diff --git a/static/js/EditFeed.js b/static/js/EditFeed.js index a68c611..7e01e90 100644 --- a/static/js/EditFeed.js +++ b/static/js/EditFeed.js @@ -67,4 +67,15 @@ function RSSTootalizerOnReady(){ for (i=0; i +
+ +

Toot Digest

+
+ For highly active feeds you may want to toot multiple posts together rather than separately avoiding a flood.
+ To avoid repetition of 'Format' for each post within a digest, use the Signature field for information that should appear only once i.e. hashtags.
+ " name="digestlimit"> posts will be combined into a digest.
+ " name="digestsig" class="btn-block">
+ + " href="#">Disable" href="#">Enable
+
+
+ From dc39b094d66fa1592ac2b0112d4ec74066c59ce0 Mon Sep 17 00:00:00 2001 From: mj-saunders Date: Mon, 4 Jun 2018 13:26:16 +0000 Subject: [PATCH 3/3] Update backend, allow tooting of digests A bit of a messy solution but should suffice for now. - Preserves url's from being truncated - Preserves other format text not directly replaced by a placeholder Also tweaked some frontend text on Edit Feed page. --- cronjob.pl | 240 +++++++++++++++++++++++++-------- static/templates/EditFeed.html | 4 +- 2 files changed, 189 insertions(+), 55 deletions(-) diff --git a/cronjob.pl b/cronjob.pl index b79c185..172ffa7 100755 --- a/cronjob.pl +++ b/cronjob.pl @@ -11,6 +11,7 @@ use RSSTootalizer::Entry; use RSSTootalizer::DB; my $VERBOSE = 1; +my $DEBUG = 1; our $config = ""; open CONFIG, "rsstootalizer.conf.json" or die "Cannot open rsstootalizer.conf.json"; @@ -39,6 +40,8 @@ binmode STDOUT, ":utf8"; if ($VERBOSE) {print STDOUT "Checking for new entries\n";} my $new_entries = 0; +my $std_length = 500; +my $link_disp_len = 23; my @feeds = RSSTootalizer::Feed->all(); # For each Feed stored in database @@ -48,6 +51,8 @@ FEED: foreach my $feed (@feeds){ # If enabled, fetch the RSS xml my $entries = $feed->fetch_entries(); next FEED unless $entries; + + my @posts; # For each entry in the xml file ENTRY: foreach my $entry ($entries->items){ # Does the entry already exist in the database? @@ -64,71 +69,72 @@ FEED: foreach my $feed (@feeds){ $entry{id} = $entry->id(); $entry{tags} = join(", ", $entry->tags()); - my $do_post = 0; my @filters = $feed->filters(); + # White/black-list entry based on filter(s) foreach my $filter (@filters){ if ($filter->apply($entry)){ if ($filter->{data}->{type} eq "white"){ - $do_post = 1; - } else { - $do_post = 0; + push(@posts, {%entry}); } } } + } # ENTRY - if ($do_post){ - my $user = $feed->user(); - my $status = $feed->{data}->{format}; - $status =~ s/{ID}/$entry{id}/g; - if (defined($entry{title})){ - $status =~ s/{Title}/$entry{title}/g; - } else { - $status =~ s/{Title}/No Title/g; - } - $status =~ s/{Link}/$entry{link}/g; - $status =~ s/{Content}/$entry{content}/g; - $status =~ s/{Author}/$entry{author}/g; - $status =~ s/{Issued}/$entry{issued}/g; - $status =~ s/{Modified}/$entry{modified}/g; - $status =~ s/{Tags}/$entry{tags}/g; + if (@posts){ + my $user = $feed->user(); + my %toot; + $toot{visibility} = 'public'; + # Visibility of a toot can be 'direct', 'private', 'unlisted' or 'public' + # 'direct' and 'unlisted' are irrelevant, except perhaps for testing + # 'private' posts only to followers [default] + # 'public' posts to public timelines [ethical issue?] + # [* Should be set per feed in the sql db *] + # [* Hardcoded here temporarily for testing *] - my %data; - if (length($status) > 500){ - $status =~ s/^(.{497}).*$/$1.../g; - } - $data{status} = $status; - - # Visibility of a toot can be 'direct', 'private', 'unlisted' or 'public' - # 'direct' and 'unlisted' are irrelevant - # 'private' posts only to followers [default] - # 'public' posts to public timelines [ethical issue?] - # [* Should be set per feed in the sql db *] - # [* Hardcoded here temporarily for testing *] - my $visibility = 'public'; - $data{visibility} = $visibility; - - $ENV{status} = encode_json({%data}); - - # encode_json breaks '\n' chars - turns them into '\\n' - # Fix them - $ENV{status} =~ s/\\\\n/\\n/g; - - open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |"); - my $reply = ""; - { - local $/ = undef; - $reply = ; - } - - $new_entries += 1; + if ($DEBUG){ + $toot{spoiler_text} = 'testing toot bot'; } - my %ne; - $ne{feed_id} = $feed->{data}->{ID}; - $ne{entry_link} = $entry{link}; - RSSTootalizer::Entry->create(%ne); - } -} + if ($feed->{data}->{digest_enabled}){ + my $digest_count = 1; + my $digest_sig = $feed->{data}->{digest_signature}; + my $post_limit = $feed->{data}->{digest_limit}; + + while (@posts){ + my $post_count = scalar(@posts); + if ( $post_count < $post_limit ){ + $post_limit = $post_count; + } + + my $snip_length = int( ($std_length - length($digest_sig)) / $post_limit ); + + my $status; + for (my $j=0; $j<$post_limit; $j++){ + my $format = $feed->{data}->{format}; + my %post = %{shift @posts}; + $status = $status . prep_post(\%post, $format, $snip_length); + $new_entries += 1; + update_db($feed, $post{link}); + } + $status = $status . $digest_sig; + $toot{status} = $status; + + send_post(\%toot, $user); + } # while @posts + } else { + # Post standard toot + while (@posts){ + my %post = %{shift @posts}; + my $format = $feed->{data}->{format}; + my $status = prep_post(\%post, $format, $std_length); + $new_entries += 1; + + send_post(\%toot, $user); + update_db($feed, $post{link}); + } + } + } # posts +} # FEED RSSTootalizer::DB->doUPDATE("UPDATE `users` SET session_id = 'invalid' WHERE TIME_TO_SEC(NOW()) - TIME_TO_SEC(`valid_from`) > 60*60*4;"); # invalidate old sessions @@ -136,3 +142,129 @@ if ($VERBOSE) { $new_entries ? ($new_entries > 1 ? print "$new_entries new entries\n" : print "$new_entries new entry\n") : print "No new entries\n"; print STDOUT "Done\n"; } + + +### FUNCTIONS ### + +sub prep_post { + # Get passed parameters + my ($postRef, $status, $snip_length) = @_; + my %post = %{$postRef}; + + if(!defined($post{title})){ + $post{title} = "No Title"; + } + + my @placeholders = qw( {ID} {Title} {Link} {Content} {Author} {Issued} {Modified} {Tags} ); + # Truncation allowances: + # * ID and Link must be preserved if present as they are url's * + # only requires allocation of 23 chars max, more chars will be ignored + # All other text will be truncated as a whole, but with respect to any static + # characters within 'Format' text + + my %status_contains; + #my $raw_content_length = 0; + my $reserved = 0; + my $reserve_count = 0; + foreach (@placeholders){ + my $rawpos = index($status, $_); + if ($rawpos != -1){ + # Store position of placeholder in status string + # and length of the text that will replace the placeholder + my $content_tag = lc(substr($_,1,-1)); + my $content_len = length($post{$content_tag}); + $status_contains{$_} = [$rawpos, $content_len, $content_tag]; + + # Remove the processed placeholder so that stored positions + # are relative to static text within status + $status =~ s/$_//g; + + if ($_ eq "{ID}" or $_ eq "{Link}"){ + # Reserve space in final post for any url's + $reserved += $content_len > 23 ? 23 : $content_len; + $reserve_count += 1; + }# else { + # Sum the length required by the text that shall + # replace any placeholders + # $raw_content_length += $content_len; + #} + } + } + + my $trunc_length = $snip_length - $reserved - length($status) - 3; #The 3 is for "..." + my $content_length_sum = 0; + my $pos_shift = 0; + my $reserved_only = 0; #Boolean + HOLDER: foreach my $holder (sort { $status_contains{$a}[0] <=> $status_contains{$b}[0] } keys %status_contains){ + my $pos = $status_contains{$holder}[0]; + my $content_length = $status_contains{$holder}[1]; + my $content_tag = $status_contains{$holder}[2]; + + # ID and Link, being reserved cases, are not measured + if ($holder ne "{Link}" and $holder ne "{ID}"){ + $content_length_sum += $content_length; + # Check if content length is within bounds + if (!$reserved_only){ + if ($content_length_sum < $trunc_length){ + # Add content text to final status) + substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} ); + $pos_shift += $content_length; + } else { + # Remove necessary chars, add three dots + my $overflow = $content_length_sum - $trunc_length; + my $remaining = $content_length - $overflow; + + # Truncate latest content piece + $post{$content_tag} = substr($post{$content_tag}, 0, $remaining) . "..."; + substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} ); + $pos_shift += ($remaining + 3); + # Prevent further unreserved additions + $reserved_only = 1; + } + } + } else { + # Add content text to final status + substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} ); + $pos_shift += $content_length; + } + + } + + #print STDOUT "$status\n"; + return $status; +} + + +sub send_post { + # Get passed parameters + my ($tootRef, $user) = @_; + my %toot = %{$tootRef}; + + $ENV{status} = encode_json({%toot}); + + # encode_json breaks '\n' chars - turns them into '\\n' + # Fix them + $ENV{status} =~ s/\\\\n/\\n/g; + #print STDOUT "$ENV{status}\n"; + + #print STDOUT "Upload - $user->{data}->{access_token} : $user->{data}->{instance}\n"; + open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |"); + my $reply = ""; + { + local $/ = undef; + $reply = ; + } +} + + +sub update_db { + # Get passed parameteres + my ($feed, $link) = @_; + + my %ne; + $ne{feed_id} = $feed->{data}->{ID}; + $ne{entry_link} = $link; + + #print STDOUT "Update db - Feed $ne{feed_id} : $link\n"; + RSSTootalizer::Entry->create(%ne); +} diff --git a/static/templates/EditFeed.html b/static/templates/EditFeed.html index 58eab6c..91fecc7 100644 --- a/static/templates/EditFeed.html +++ b/static/templates/EditFeed.html @@ -44,6 +44,7 @@

Format

Enter the format for the toot. You can use the placeholders {ID}, {Title}, {Link}, {Content}, {Author}, {Issued}, {Modified} and {Tags} to substitute the respective values.
+ Anything greater than 500 chars will be truncated. URL's shall be preserved.
" name="format" class="btn-block">
@@ -76,10 +77,11 @@
For highly active feeds you may want to toot multiple posts together rather than separately avoiding a flood.
To avoid repetition of 'Format' for each post within a digest, use the Signature field for information that should appear only once i.e. hashtags.
+ Placeholder text shall be truncated if necessary. URL's shall be preserved. Any other format text will remain.
" name="digestlimit"> posts will be combined into a digest.
" name="digestsig" class="btn-block">
- " href="#">Disable" href="#">Enable
+ " href="#">Disable" href="#">Enable