Merge remote-tracking branch 'devel-test/develop' into develop
commit
e9d45023c4
|
|
@ -63,6 +63,20 @@ sub fill_content {
|
|||
}
|
||||
}
|
||||
|
||||
if ($main::FORM{enabledigest} and "x".$main::FORM{enabledigest} eq "x".$feed->{data}->{ID}){
|
||||
$feed->{data}->{digest_enabled} = 1;
|
||||
$feed->save();
|
||||
}
|
||||
if ($main::FORM{disabledigest} and "x".$main::FORM{disabledigest} eq "x".$feed->{data}->{ID}){
|
||||
$feed->{data}->{digest_enabled} = 0;
|
||||
$feed->save();
|
||||
}
|
||||
if ($main::FORM{action} and "x".$main::FORM{action} eq "xsavedigest"){
|
||||
$feed->{data}->{digest_limit} = $main::FORM{digestlimit};
|
||||
$feed->{data}->{digest_signature} = $main::FORM{digestsig};
|
||||
$feed->save();
|
||||
}
|
||||
|
||||
my @param_entries;
|
||||
my @filters = $feed->filters();
|
||||
my $feeddata = $feed->fetch_entries();
|
||||
|
|
@ -106,6 +120,10 @@ sub fill_content {
|
|||
$output->param("url", $feed->{data}->{url});
|
||||
$output->param("feed_id", $feed->{data}->{ID});
|
||||
$output->param("format", $feed->{data}->{format});
|
||||
|
||||
$output->param("digestenabled", $feed->{data}->{digest_enabled});
|
||||
$output->param("digestlimit", $feed->{data}->{digest_limit});
|
||||
$output->param("digestsig", $feed->{data}->{digest_signature});
|
||||
return 1;
|
||||
}
|
||||
sub prerender {
|
||||
|
|
|
|||
244
cronjob.pl
244
cronjob.pl
|
|
@ -11,6 +11,7 @@ use RSSTootalizer::Entry;
|
|||
use RSSTootalizer::DB;
|
||||
|
||||
my $VERBOSE = 1;
|
||||
my $DEBUG = 1;
|
||||
|
||||
our $config = "";
|
||||
open CONFIG, "rsstootalizer.conf.json" or die "Cannot open rsstootalizer.conf.json";
|
||||
|
|
@ -39,14 +40,22 @@ binmode STDOUT, ":utf8";
|
|||
|
||||
if ($VERBOSE) {print STDOUT "Checking for new entries\n";}
|
||||
my $new_entries = 0;
|
||||
my $std_length = 500;
|
||||
my $link_disp_len = 23;
|
||||
|
||||
my @feeds = RSSTootalizer::Feed->all();
|
||||
# For each Feed stored in database
|
||||
FEED: foreach my $feed (@feeds){
|
||||
next FEED unless $feed;
|
||||
next FEED unless $feed->{data}->{enabled};
|
||||
# If enabled, fetch the RSS xml
|
||||
my $entries = $feed->fetch_entries();
|
||||
next FEED unless $entries;
|
||||
|
||||
my @posts;
|
||||
# For each entry in the xml file
|
||||
ENTRY: foreach my $entry ($entries->items){
|
||||
# Does the entry already exist in the database?
|
||||
my @seen_entries = $feed->entry_by("entry_link", $entry->link());
|
||||
next ENTRY if ((scalar @seen_entries) > 0);
|
||||
|
||||
|
|
@ -60,71 +69,72 @@ FEED: foreach my $feed (@feeds){
|
|||
$entry{id} = $entry->id();
|
||||
$entry{tags} = join(", ", $entry->tags());
|
||||
|
||||
my $do_post = 0;
|
||||
my @filters = $feed->filters();
|
||||
# White/black-list entry based on filter(s)
|
||||
foreach my $filter (@filters){
|
||||
if ($filter->apply($entry)){
|
||||
if ($filter->{data}->{type} eq "white"){
|
||||
$do_post = 1;
|
||||
} else {
|
||||
$do_post = 0;
|
||||
push(@posts, {%entry});
|
||||
}
|
||||
}
|
||||
}
|
||||
} # ENTRY
|
||||
|
||||
if ($do_post){
|
||||
my $user = $feed->user();
|
||||
my $status = $feed->{data}->{format};
|
||||
$status =~ s/{ID}/$entry{id}/g;
|
||||
if (defined($entry{title})){
|
||||
$status =~ s/{Title}/$entry{title}/g;
|
||||
} else {
|
||||
$status =~ s/{Title}/No Title/g;
|
||||
}
|
||||
$status =~ s/{Link}/$entry{link}/g;
|
||||
$status =~ s/{Content}/$entry{content}/g;
|
||||
$status =~ s/{Author}/$entry{author}/g;
|
||||
$status =~ s/{Issued}/$entry{issued}/g;
|
||||
$status =~ s/{Modified}/$entry{modified}/g;
|
||||
$status =~ s/{Tags}/$entry{tags}/g;
|
||||
if (@posts){
|
||||
my $user = $feed->user();
|
||||
my %toot;
|
||||
$toot{visibility} = 'public';
|
||||
# Visibility of a toot can be 'direct', 'private', 'unlisted' or 'public'
|
||||
# 'direct' and 'unlisted' are irrelevant, except perhaps for testing
|
||||
# 'private' posts only to followers [default]
|
||||
# 'public' posts to public timelines [ethical issue?]
|
||||
# [* Should be set per feed in the sql db *]
|
||||
# [* Hardcoded here temporarily for testing *]
|
||||
|
||||
my %data;
|
||||
if (length($status) > 500){
|
||||
$status =~ s/^(.{497}).*$/$1.../g;
|
||||
}
|
||||
$data{status} = $status;
|
||||
|
||||
# Visibility of a toot can be 'direct', 'private', 'unlisted' or 'public'
|
||||
# 'direct' and 'unlisted' are irrelevant
|
||||
# 'private' posts only to followers [default]
|
||||
# 'public' posts to public timelines [ethical issue?]
|
||||
# [* Should be set per feed in the sql db *]
|
||||
# [* Hardcoded here temporarily for testing *]
|
||||
my $visibility = 'public';
|
||||
$data{visibility} = $visibility;
|
||||
|
||||
$ENV{status} = encode_json({%data});
|
||||
|
||||
# encode_json breaks '\n' chars - turns them into '\\n'
|
||||
# Fix them
|
||||
$ENV{status} =~ s/\\\\n/\\n/g;
|
||||
|
||||
open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |");
|
||||
my $reply = "";
|
||||
{
|
||||
local $/ = undef;
|
||||
$reply = <DATA>;
|
||||
}
|
||||
|
||||
$new_entries += 1;
|
||||
if ($DEBUG){
|
||||
$toot{spoiler_text} = 'testing toot bot';
|
||||
}
|
||||
|
||||
my %ne;
|
||||
$ne{feed_id} = $feed->{data}->{ID};
|
||||
$ne{entry_link} = $entry{link};
|
||||
RSSTootalizer::Entry->create(%ne);
|
||||
}
|
||||
}
|
||||
if ($feed->{data}->{digest_enabled}){
|
||||
my $digest_count = 1;
|
||||
my $digest_sig = $feed->{data}->{digest_signature};
|
||||
my $post_limit = $feed->{data}->{digest_limit};
|
||||
|
||||
while (@posts){
|
||||
my $post_count = scalar(@posts);
|
||||
if ( $post_count < $post_limit ){
|
||||
$post_limit = $post_count;
|
||||
}
|
||||
|
||||
my $snip_length = int( ($std_length - length($digest_sig)) / $post_limit );
|
||||
|
||||
my $status;
|
||||
for (my $j=0; $j<$post_limit; $j++){
|
||||
my $format = $feed->{data}->{format};
|
||||
my %post = %{shift @posts};
|
||||
$status = $status . prep_post(\%post, $format, $snip_length);
|
||||
$new_entries += 1;
|
||||
update_db($feed, $post{link});
|
||||
}
|
||||
$status = $status . $digest_sig;
|
||||
$toot{status} = $status;
|
||||
|
||||
send_post(\%toot, $user);
|
||||
} # while @posts
|
||||
} else {
|
||||
# Post standard toot
|
||||
while (@posts){
|
||||
my %post = %{shift @posts};
|
||||
my $format = $feed->{data}->{format};
|
||||
my $status = prep_post(\%post, $format, $std_length);
|
||||
$new_entries += 1;
|
||||
|
||||
send_post(\%toot, $user);
|
||||
update_db($feed, $post{link});
|
||||
}
|
||||
}
|
||||
} # posts
|
||||
} # FEED
|
||||
|
||||
RSSTootalizer::DB->doUPDATE("UPDATE `users` SET session_id = 'invalid' WHERE TIME_TO_SEC(NOW()) - TIME_TO_SEC(`valid_from`) > 60*60*4;"); # invalidate old sessions
|
||||
|
||||
|
|
@ -132,3 +142,129 @@ if ($VERBOSE) {
|
|||
$new_entries ? ($new_entries > 1 ? print "$new_entries new entries\n" : print "$new_entries new entry\n") : print "No new entries\n";
|
||||
print STDOUT "Done\n";
|
||||
}
|
||||
|
||||
|
||||
### FUNCTIONS ###
|
||||
|
||||
sub prep_post {
|
||||
# Get passed parameters
|
||||
my ($postRef, $status, $snip_length) = @_;
|
||||
my %post = %{$postRef};
|
||||
|
||||
if(!defined($post{title})){
|
||||
$post{title} = "No Title";
|
||||
}
|
||||
|
||||
my @placeholders = qw( {ID} {Title} {Link} {Content} {Author} {Issued} {Modified} {Tags} );
|
||||
# Truncation allowances:
|
||||
# * ID and Link must be preserved if present as they are url's *
|
||||
# only requires allocation of 23 chars max, more chars will be ignored
|
||||
# All other text will be truncated as a whole, but with respect to any static
|
||||
# characters within 'Format' text
|
||||
|
||||
my %status_contains;
|
||||
#my $raw_content_length = 0;
|
||||
my $reserved = 0;
|
||||
my $reserve_count = 0;
|
||||
foreach (@placeholders){
|
||||
my $rawpos = index($status, $_);
|
||||
if ($rawpos != -1){
|
||||
# Store position of placeholder in status string
|
||||
# and length of the text that will replace the placeholder
|
||||
my $content_tag = lc(substr($_,1,-1));
|
||||
my $content_len = length($post{$content_tag});
|
||||
$status_contains{$_} = [$rawpos, $content_len, $content_tag];
|
||||
|
||||
# Remove the processed placeholder so that stored positions
|
||||
# are relative to static text within status
|
||||
$status =~ s/$_//g;
|
||||
|
||||
if ($_ eq "{ID}" or $_ eq "{Link}"){
|
||||
# Reserve space in final post for any url's
|
||||
$reserved += $content_len > 23 ? 23 : $content_len;
|
||||
$reserve_count += 1;
|
||||
}# else {
|
||||
# Sum the length required by the text that shall
|
||||
# replace any placeholders
|
||||
# $raw_content_length += $content_len;
|
||||
#}
|
||||
}
|
||||
}
|
||||
|
||||
my $trunc_length = $snip_length - $reserved - length($status) - 3; #The 3 is for "..."
|
||||
my $content_length_sum = 0;
|
||||
my $pos_shift = 0;
|
||||
my $reserved_only = 0; #Boolean
|
||||
HOLDER: foreach my $holder (sort { $status_contains{$a}[0] <=> $status_contains{$b}[0] } keys %status_contains){
|
||||
my $pos = $status_contains{$holder}[0];
|
||||
my $content_length = $status_contains{$holder}[1];
|
||||
my $content_tag = $status_contains{$holder}[2];
|
||||
|
||||
# ID and Link, being reserved cases, are not measured
|
||||
if ($holder ne "{Link}" and $holder ne "{ID}"){
|
||||
$content_length_sum += $content_length;
|
||||
# Check if content length is within bounds
|
||||
if (!$reserved_only){
|
||||
if ($content_length_sum < $trunc_length){
|
||||
# Add content text to final status)
|
||||
substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
|
||||
$pos_shift += $content_length;
|
||||
} else {
|
||||
# Remove necessary chars, add three dots
|
||||
my $overflow = $content_length_sum - $trunc_length;
|
||||
my $remaining = $content_length - $overflow;
|
||||
|
||||
# Truncate latest content piece
|
||||
$post{$content_tag} = substr($post{$content_tag}, 0, $remaining) . "...";
|
||||
substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
|
||||
$pos_shift += ($remaining + 3);
|
||||
# Prevent further unreserved additions
|
||||
$reserved_only = 1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
# Add content text to final status
|
||||
substr( $status, ($pos + $pos_shift), 0, $post{$content_tag} );
|
||||
$pos_shift += $content_length;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#print STDOUT "$status\n";
|
||||
return $status;
|
||||
}
|
||||
|
||||
|
||||
sub send_post {
|
||||
# Get passed parameters
|
||||
my ($tootRef, $user) = @_;
|
||||
my %toot = %{$tootRef};
|
||||
|
||||
$ENV{status} = encode_json({%toot});
|
||||
|
||||
# encode_json breaks '\n' chars - turns them into '\\n'
|
||||
# Fix them
|
||||
$ENV{status} =~ s/\\\\n/\\n/g;
|
||||
#print STDOUT "$ENV{status}\n";
|
||||
|
||||
#print STDOUT "Upload - $user->{data}->{access_token} : $user->{data}->{instance}\n";
|
||||
open(DATA, "./post_status.bash '$user->{data}->{access_token}' '$user->{data}->{instance}' |");
|
||||
my $reply = "";
|
||||
{
|
||||
local $/ = undef;
|
||||
$reply = <DATA>;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub update_db {
|
||||
# Get passed parameteres
|
||||
my ($feed, $link) = @_;
|
||||
|
||||
my %ne;
|
||||
$ne{feed_id} = $feed->{data}->{ID};
|
||||
$ne{entry_link} = $link;
|
||||
|
||||
#print STDOUT "Update db - Feed $ne{feed_id} : $link\n";
|
||||
RSSTootalizer::Entry->create(%ne);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
ALTER TABLE `feeds`
|
||||
ADD `digest_enabled` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 AFTER `enabled`,
|
||||
ADD `digest_limit` tinyint UNSIGNED NOT NULL DEFAULT 5 AFTER `digest_enabled`,
|
||||
ADD `digest_signature` varchar(100) AFTER `digest_limit`;
|
||||
|
||||
|
|
@ -4,3 +4,8 @@
|
|||
.red {
|
||||
background-color: #FF8080;
|
||||
}
|
||||
|
||||
#diglim {
|
||||
font-family: Consolas, Lucida Console, monospace;
|
||||
width: 3em;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,4 +67,15 @@ function RSSTootalizerOnReady(){
|
|||
for (i=0; i<filters.length; i++){
|
||||
appendFilter(filters[i].ID, filters[i].field, filters[i].regex, filters[i].type, filters[i].match);
|
||||
}
|
||||
|
||||
$("a.enabledigest").on("click", function(){
|
||||
var btn = $(this);
|
||||
var l = "index.pl?mode=EditFeed&id="+btn.data("id")+"&enabledigest="+btn.data("id");
|
||||
document.location.href=l;
|
||||
});
|
||||
$("a.disabledigest").on("click", function(){
|
||||
var btn = $(this);
|
||||
var l = "index.pl?mode=EditFeed&id="+btn.data("id")+"&disabledigest="+btn.data("id");
|
||||
document.location.href=l;
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@
|
|||
<h3>Format</h3>
|
||||
<div id="format">
|
||||
Enter the format for the toot. You can use the placeholders <code>{ID}</code>, <code>{Title}</code>, <code>{Link}</code>, <code>{Content}</code>, <code>{Author}</code>, <code>{Issued}</code>, <code>{Modified}</code> and <code>{Tags}</code> to substitute the respective values. <br />
|
||||
<i>Anything greater than 500 chars will be truncated. URL's shall be preserved.</i><br />
|
||||
<input type="text" placeholder="{Title} - {Link} by {Author} -- posted at {Issued} with #RSSTootalizer" maxlength=500 value="<TMPL_VAR NAME="format">" name="format" class="btn-block"><br />
|
||||
<input type="Submit" value="Save format" class="btn btn-primary btn-block">
|
||||
</div>
|
||||
|
|
@ -70,6 +71,20 @@
|
|||
</div>
|
||||
</form>
|
||||
|
||||
<form id="form_digest" method="POST">
|
||||
<input type="hidden" name="action" value="savedigest">
|
||||
<h3>Toot Digest</h3>
|
||||
<div>
|
||||
For highly active feeds you may want to toot multiple posts together rather than separately avoiding a flood.<br />
|
||||
<i>To avoid repetition of 'Format' for each post within a digest, use the Signature field for information that should appear only once i.e. hashtags.</i><br />
|
||||
<i>Placeholder text shall be truncated if necessary. URL's shall be preserved. Any other format text will remain.</i><br />
|
||||
<input type="text" id="diglim" maxlength=2 value="<TMPL_VAR NAME="digestlimit">" name="digestlimit"> posts will be combined into a digest.<br />
|
||||
<input type="text" placeholder="Signed #RSSTootalizer" maxlength=100 value="<TMPL_VAR NAME="digestsig">" name="digestsig" class="btn-block"><br />
|
||||
<input type="Submit" value="Save digest" class="btn btn-primary">
|
||||
<TMPL_IF NAME="digestenabled"><a class="btn btn-danger disabledigest" data-id="<TMPL_VAR NAME="feed_id">" href="#">Disable</a><TMPL_ELSE><a class="btn btn-default enabledigest" data-id="<TMPL_VAR NAME="feed_id">" href="#">Enable</a></TMPL_IF>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ binmode STDOUT, ":utf8";
|
|||
my @migrations = glob ("migrations/*sql");
|
||||
foreach my $migration (@migrations){
|
||||
my $sth = RSSTootalizer::DB->doSELECT("SELECT * FROM migrations WHERE name = ?", $migration);
|
||||
if (scalar(@$sth) == 0){
|
||||
if ( scalar(@$sth) == 0 ){
|
||||
print "Running migration $migration\n";
|
||||
open (M, "<", $migration);
|
||||
my $sql = "";
|
||||
|
|
|
|||
Loading…
Reference in New Issue