Disable verbose and debug for production

A bit of a messy solution but should suffice for now.
- Preserves url's from being truncated
- Preserves other format text not directly replaced by a placeholder

Also tweaked some frontend text on Edit Feed page.

RSSTootalizer/Website/EditFeed.pm

@@ -63,6 +63,20 @@ sub fill_content {
 		}
 	}
 
+	if ($main::FORM{enabledigest} and "x".$main::FORM{enabledigest} eq "x".$feed->{data}->{ID}){
+		$feed->{data}->{digest_enabled} = 1;
+		$feed->save();
+	}
+	if ($main::FORM{disabledigest} and "x".$main::FORM{disabledigest} eq "x".$feed->{data}->{ID}){
+		$feed->{data}->{digest_enabled} = 0;
+		$feed->save();
+	}
+	if ($main::FORM{action} and "x".$main::FORM{action} eq "xsavedigest"){
+		$feed->{data}->{digest_limit} = $main::FORM{digestlimit};
+		$feed->{data}->{digest_signature} = $main::FORM{digestsig};
+		$feed->save();
+	}
+
 	my @param_entries;
 	my @filters = $feed->filters();
 	my $feeddata = $feed->fetch_entries();
@@ -106,6 +120,10 @@ sub fill_content {
 	$output->param("url", $feed->{data}->{url});
 	$output->param("feed_id", $feed->{data}->{ID});
 	$output->param("format", $feed->{data}->{format});
+
+	$output->param("digestenabled", $feed->{data}->{digest_enabled});
+	$output->param("digestlimit", $feed->{data}->{digest_limit});
+	$output->param("digestsig", $feed->{data}->{digest_signature});
 	return 1;
 }
 sub prerender {

cronjob.pl

@@ -10,6 +10,9 @@ use RSSTootalizer::User;
 use RSSTootalizer::Entry;
 use RSSTootalizer::DB;
 
+my $VERBOSE = 0;
+my $DEBUG = 0;
+
 our $config = "";
 open CONFIG, "rsstootalizer.conf.json" or die "Cannot open rsstootalizer.conf.json";
 {
@@ -35,13 +38,24 @@ sub Error {{{
 binmode STDERR, ":utf8";
 binmode STDOUT, ":utf8";
 
+if ($VERBOSE) {print STDOUT "Checking for new entries\n";}
+my $new_entries = 0;
+my $std_length = 500;
+my $link_disp_len = 23;
+
 my @feeds = RSSTootalizer::Feed->all();
+# For each Feed stored in database
 FEED: foreach my $feed (@feeds){
 	next FEED unless $feed;
 	next FEED unless $feed->{data}->{enabled};
+	# If enabled, fetch the RSS xml
 	my $entries = $feed->fetch_entries();
 	next FEED unless $entries;
+
+	my @posts;
+	# For each entry in the xml file
 	ENTRY: foreach my $entry ($entries->items){
+		# Does the entry already exist in the database?
 		my @seen_entries = $feed->entry_by("entry_link", $entry->link());
 		next ENTRY if ((scalar @seen_entries) > 0);
 
@@ -55,54 +69,199 @@ FEED: foreach my $feed (@feeds){
 		$entry{id} = $entry->id();
 		$entry{tags} = join(", ", $entry->tags());
 
-		my $do_post = 0;
 		my @filters = $feed->filters();
+		# White/black-list entry based on filter(s)
 		foreach my $filter (@filters){
 			if ($filter->apply($entry)){
 				if ($filter->{data}->{type} eq "white"){
-					$do_post = 1;
-				} else {
-					$do_post = 0;
+					push(@posts, {%entry});
 				}
 			}
 		}
+	} # ENTRY
 
-		if ($do_post){
-			my $user = $feed->user();
-			my $status = $feed->{data}->{format};
-			$status =~ s/{ID}/$entry{id}/g;
-			if (defined($entry{title})){
-				$status =~ s/{Title}/$entry{title}/g;
-			} else {
-				$status =~ s/{Title}/No Title/g;
-			}
-			$status =~ s/{Link}/$entry{link}/g;
-			$status =~ s/{Content}/$entry{content}/g;
-			$status =~ s/{Author}/$entry{author}/g;
-			$status =~ s/{Issued}/$entry{issued}/g;
-			$status =~ s/{Modified}/$entry{modified}/g;
-			$status =~ s/{Tags}/$entry{tags}/g;
+	if (@posts){
+		my $user = $feed->user();
+		my %toot;
+		$toot{visibility} = 'public';
+		# Visibility of a toot can be 'direct', 'private', 'unlisted' or 'public'
+		# 'direct' and 'unlisted' are irrelevant, except perhaps for testing
+		# 'private' posts only to followers [default]
+		# 'public' posts to public timelines [ethical issue?]
+		# [* Should be set per feed in the sql db *]
+		# [* Hardcoded here temporarily for testing *]
 
-			my %data;
-			if (length($status) > 500){
-				$status =~ s/^(.{497}).*$/$1.../g;
-			}
-			$data{status} = $status;
-			$ENV{status} = encode_json({%data});
-
-			open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |");
-			my $reply = "";
-			{
-				local $/ = undef;
-				$reply = <DATA>;
-			}
+		if ($DEBUG){
+			$toot{spoiler_text} = 'testing toot bot';
 		}
 
-		my %ne;
-		$ne{feed_id} = $feed->{data}->{ID};
-		$ne{entry_link} = $entry{link};
-		RSSTootalizer::Entry->create(%ne);
+		if ($feed->{data}->{digest_enabled}){
+			my $digest_count = 1;
+			my $digest_sig = $feed->{data}->{digest_signature};
+			my $post_limit = $feed->{data}->{digest_limit};
+
+			while (@posts){
+				my $post_count = scalar(@posts);
+				if ( $post_count < $post_limit ){
+					$post_limit = $post_count;
+				}
+
+				my $snip_length = int( ($std_length - length($digest_sig)) / $post_limit );
+
+				my $status;
+				for (my $j=0; $j<$post_limit; $j++){
+					my $format = $feed->{data}->{format};
+					my %post = %{shift @posts};
+					$status = $status . prep_post(\%post, $format, $snip_length);
+					$new_entries += 1;
+					update_db($feed, $post{link});
+				}
+				$status = $status . $digest_sig;
+				$toot{status} = $status;
+
+				send_post(\%toot, $user);
+			} # while @posts
+		} else {
+			# Post standard toot
+			while (@posts){
+				my %post = %{shift @posts};
+				my $format = $feed->{data}->{format};
+				my $status = prep_post(\%post, $format, $std_length);
+				$new_entries += 1;
+
+				send_post(\%toot, $user);
+				update_db($feed, $post{link});
+			}
+		}
+	} # posts
+} # FEED
+
+RSSTootalizer::DB->doUPDATE("UPDATE `users` SET session_id = 'invalid' WHERE TIME_TO_SEC(NOW()) - TIME_TO_SEC(`valid_from`) > 60*60*4;"); # invalidate old sessions
+
+if ($VERBOSE) {
+	$new_entries ? ($new_entries > 1 ? print "$new_entries new entries\n" : print "$new_entries new entry\n") : print "No new entries\n";
+	print STDOUT "Done\n";
+}
+
+
+### FUNCTIONS ###
+
+sub prep_post {
+	# Get passed parameters
+	my ($postRef, $status, $snip_length) = @_;
+	my %post = %{$postRef};
+
+	if(!defined($post{title})){
+		$post{title} = "No Title";
+	}
+
+	my @placeholders = qw( {ID} {Title} {Link} {Content} {Author} {Issued} {Modified} {Tags} );
+	# Truncation allowances:
+	# * ID and Link must be preserved if present as they are url's *
+	#     only requires allocation of 23 chars max, more chars will be ignored
+	# All other text will be truncated as a whole, but with respect to any static
+	# characters within 'Format' text
+
+	my %status_contains;
+	#my $raw_content_length = 0;
+	my $reserved = 0;
+	my $reserve_count = 0;
+	foreach (@placeholders){
+		my $rawpos = index($status, $_);
+		if ($rawpos != -1){
+			# Store position of placeholder in status string
+			# and length of the text that will replace the placeholder
+			my $content_tag = lc(substr($_,1,-1));
+			my $content_len = length($post{$content_tag});
+			$status_contains{$_} = [$rawpos, $content_len, $content_tag];
+
+			# Remove the processed placeholder so that stored positions
+			# are relative to static text within status
+			$status =~ s/$_//g;
+
+			if ($_ eq "{ID}" or $_ eq "{Link}"){
+				# Reserve space in final post for any url's
+				$reserved += $content_len > 23 ? 23 : $content_len;
+				$reserve_count += 1;
+			}# else {
+				# Sum the length required by the text that shall
+				# replace any placeholders
+			#	$raw_content_length += $content_len;
+			#}
+		}
+	}
+
+	my $trunc_length = $snip_length - $reserved - length($status) - 3; #The 3 is for "..."
+	my $content_length_sum = 0;
+	my $pos_shift = 0;
+	my $reserved_only = 0; #Boolean
+	HOLDER: foreach my $holder (sort { $status_contains{$a}[0] <=> $status_contains{$b}[0] } keys %status_contains){
+		my $pos = $status_contains{$holder}[0];
+		my $content_length = $status_contains{$holder}[1];
+		my $content_tag = $status_contains{$holder}[2];
+
+		# ID and Link, being reserved cases, are not measured
+		if ($holder ne "{Link}" and $holder ne "{ID}"){
+			$content_length_sum += $content_length;
+			# Check if content length is within bounds
+			if (!$reserved_only){
+				if ($content_length_sum < $trunc_length){
+					# Add content text to final status)
+					substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
+					$pos_shift += $content_length;
+				} else {
+					# Remove necessary chars, add three dots
+					my $overflow = $content_length_sum - $trunc_length;
+					my $remaining = $content_length - $overflow;
+
+					# Truncate latest content piece
+					$post{$content_tag} = substr($post{$content_tag}, 0, $remaining) . "...";
+					substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
+					$pos_shift += ($remaining + 3);
+					# Prevent further unreserved additions
+					$reserved_only = 1;
+				}
+			}
+		} else {
+			# Add content text to final status
+			substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
+			$pos_shift += $content_length;
+		}
+
+	}
+
+	#print STDOUT "$status\n";
+	return $status;
+}
+
+
+sub send_post {
+	# Get passed parameters
+	my ($tootRef, $user) = @_;
+	my %toot = %{$tootRef};
+
+	$ENV{status} = encode_json({%toot});
+
+	# encode_json breaks '\n' chars - turns them into '\\n'
+	# Fix them
+	$ENV{status} =~ s/\\\\n/\\n/g;
+
+	open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |");
+	my $reply = "";
+	{
+		local $/ = undef;
+		$reply = <DATA>;
 	}
 }
 
-RSSTootalizer::DB->doUPDATE("UPDATE `users` SET session_id = 'invalid' WHERE TIME_TO_SEC(NOW()) - TIME_TO_SEC(`valid_from`) > 60*60*4;"); # invalidate old sessions
+
+sub update_db {
+	# Get passed parameteres
+	my ($feed, $link) = @_;
+
+	my %ne;
+	$ne{feed_id} = $feed->{data}->{ID};
+	$ne{entry_link} = $link;
+
+	RSSTootalizer::Entry->create(%ne);
+}

migrations/0002-allow_toot_digests.sql

@@ -0,0 +1,5 @@
+ALTER TABLE `feeds`
+  ADD `digest_enabled` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 AFTER `enabled`,
+  ADD `digest_limit` tinyint UNSIGNED NOT NULL DEFAULT 5 AFTER `digest_enabled`,
+  ADD `digest_signature` varchar(100) AFTER `digest_limit`;
+

static/css/EditFeed.css

@@ -4,3 +4,8 @@
 .red {
 	background-color: #FF8080;
 }
+
+#diglim {
+	font-family: Consolas, Lucida Console, monospace;
+	width: 3em;
+}

static/js/EditFeed.js

@@ -67,4 +67,15 @@ function RSSTootalizerOnReady(){
 	for (i=0; i<filters.length; i++){
 		appendFilter(filters[i].ID, filters[i].field, filters[i].regex, filters[i].type, filters[i].match);
 	}
+
+	$("a.enabledigest").on("click", function(){
+                var btn = $(this);
+                var l = "index.pl?mode=EditFeed&id="+btn.data("id")+"&enabledigest="+btn.data("id");
+                document.location.href=l;
+        });
+        $("a.disabledigest").on("click", function(){
+                var btn = $(this);
+                var l = "index.pl?mode=EditFeed&id="+btn.data("id")+"&disabledigest="+btn.data("id");
+                document.location.href=l;
+        });
 }

static/templates/EditFeed.html

@@ -44,6 +44,7 @@
 			<h3>Format</h3>
 			<div id="format">
 				Enter the format for the toot. You can use the placeholders <code>{ID}</code>, <code>{Title}</code>, <code>{Link}</code>, <code>{Content}</code>, <code>{Author}</code>, <code>{Issued}</code>, <code>{Modified}</code> and <code>{Tags}</code> to substitute the respective values. <br />
+				<i>Anything greater than 500 chars will be truncated. URL's shall be preserved.</i><br />
 				<input type="text" placeholder="{Title} - {Link} by {Author} -- posted at {Issued} with #RSSTootalizer" maxlength=500 value="<TMPL_VAR NAME="format">" name="format" class="btn-block"><br />
 				<input type="Submit" value="Save format" class="btn btn-primary btn-block">
 			</div>
@@ -70,6 +71,20 @@
 			</div>
 			</form>
 
+			<form id="form_digest" method="POST">
+			<input type="hidden" name="action" value="savedigest">
+			<h3>Toot Digest</h3>
+			<div>
+				For highly active feeds you may want to toot multiple posts together rather than separately avoiding a flood.<br />
+				<i>To avoid repetition of 'Format' for each post within a digest, use the Signature field for information that should appear only once i.e. hashtags.</i><br />
+				<i>Placeholder text shall be truncated if necessary. URL's shall be preserved. Any other format text will remain.</i><br />
+				<input type="text" id="diglim" maxlength=2 value="<TMPL_VAR NAME="digestlimit">" name="digestlimit"> posts will be combined into a digest.<br />
+				<input type="text" placeholder="Signed #RSSTootalizer" maxlength=100 value="<TMPL_VAR NAME="digestsig">" name="digestsig" class="btn-block"><br />
+				<input type="Submit" value="Save digest" class="btn btn-primary">
+				<TMPL_IF NAME="digestenabled"><a class="btn btn-danger disabledigest" data-id="<TMPL_VAR NAME="feed_id">" href="#">Disable</a><TMPL_ELSE><a class="btn btn-default enabledigest" data-id="<TMPL_VAR NAME="feed_id">" href="#">Enable</a></TMPL_IF>
+			</div>
+			</form>
+
 		</div>
 	</div>
 </div>

update_db.pl

@@ -27,7 +27,7 @@ binmode STDOUT, ":utf8";
 my @migrations = glob ("migrations/*sql");
 foreach my $migration (@migrations){
 	my $sth = RSSTootalizer::DB->doSELECT("SELECT * FROM migrations WHERE name = ?", $migration);
-	if (scalar(@$sth) == 0){
+	if ( scalar(@$sth) == 0 ){
 		print "Running migration $migration\n";
 		open (M, "<", $migration);
 		my $sql = "";