Logo Search packages:      
Sourcecode: pan version File versions  Download package

score.c

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/*
 * Pan - A Newsreader for Gtk+
 * Copyright (C) 2003  Charles Kerr <charles@rebelbase.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/*********************
**********************  Includes
*********************/

#include <config.h>

#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>

#include <glib.h>

#include <pan/base/debug.h>
#include <pan/base/pan-config.h>
#include <pan/base/pan-glib-extensions.h>
#include <pan/base/pan-i18n.h>
#include <pan/base/log.h>
#include <pan/base/util-file.h>

#include <pan/filters/filter.h>
#include <pan/filters/filter-aggregate.h>
#include <pan/filters/filter-bytes.h>
#include <pan/filters/filter-date.h>
#include <pan/filters/filter-phrase.h>
#include <pan/filters/filter-linecount.h>
#include <pan/filters/score.h>

#include <pan/prefs.h>

#include <sys/types.h>

/*********************
**********************  Defines / Enumerated types
*********************/

/*********************
**********************  Macros
*********************/

/*********************
**********************  Structures / Typedefs
*********************/

typedef struct
{
      const char * name;
      FilterPhrase ** filters;
      int filter_qty;
      gboolean section_name_negate;
      GSList * scores;
}
Section;

typedef struct
{
      const char * name;
      const char * filename;
      int begin_line;
      int end_line;
      gboolean expired;

      Filter * filter;

      int value;
      gboolean value_assign_flag;
      gboolean stop_scoring;
}
Score;

/**
 * This is just used while parsing the scorefiles.
 */
00093 typedef struct
{
      /**
       * The current Section object, or NULL if none.
       */
00098       Section * current_section;

      /**
       * The current Score object, or NULL if none.
       */
00103       Score * current_score;

      /**
       * This is a reverse-order GSList of all the nested
       * FilterAggregates.  For most scores there is only
       * the one aggregate, but with slrn's "{:" notation
       * they can be nested.
       *
       * The list is kept in reverse order for convenience:
       * the first element is the bottom, so adding new
       * Score objects, and popping the stack when "}" is
       * reached, both operate on the first node instead of
       * having to walk the list.
       */
00117       GSList * aggregates;
      
      /**
       * This is the current day, represented as a ulong.
       * @see get_today
       * @see has_score_expired
       */
00124       unsigned long today;
}
ScorefileContext;

/*********************
**********************  Private Function Prototypes
*********************/

/*********************
**********************  Variables
*********************/

/***********
************  Extern
***********/

/***********
************  Public
***********/

/***********
************  Private
***********/

static GSList * _sections = NULL;
static GStaticMutex _score_mutex = G_STATIC_MUTEX_INIT;
static GStringChunk * _string_chunk = NULL;

/*********************
**********************  BEGINNING OF SOURCE
*********************/

/****
*****
*****  Private Utilities
*****
****/

static void
score_error (const char * error_message, const char * line_contents, int line_number, const char * filename)
{
      log_add_va (LOG_ERROR, _("scorefile %s, line %d: %s (%s)"),
            filename, line_number,
            error_message,
            line_contents);
}

static void
score_info (const char * message, const char * line_contents, int line_number, const char * filename)
{
      log_add_va (LOG_INFO, _("scorefile %s, line %d: %s (%s)"),
            filename, line_number,
            message,
            line_contents);
}

const char*
score_get_main_scorefile_filename (void)
{
      static char * filename = NULL;

      if (filename == NULL)
      {
            /* get the filename */
            filename = pan_config_get_string (KEY_SCOREFILE, KEY_SCOREFILE_DEFAULT);
            replace_gstr (&filename, pan_file_normalize (filename, NULL));
      }

      return filename;
}

/****
*****
*****  Date / Expiration
*****
****/

static unsigned long
get_today (void)
{
      unsigned long mm, yy, dd;
      struct tm tm_struct;
      const time_t now = time (NULL);

      pan_localtime_r (&now, &tm_struct);
      yy = tm_struct.tm_year;
      mm = tm_struct.tm_mon;
      dd = tm_struct.tm_mday;

      return yy * 10000 + mm * 100 + dd;
}

/**
 * 0 if it has not expired
 * 1 if it has expired
 * -1 if an error occurred while parsing
 */
static int
has_score_expired (unsigned const char *s, unsigned long today)
{
      unsigned long mm, dd, yyyy;
      unsigned long score_time;

      g_return_val_if_fail (is_nonempty_string(s), 0);

      if (((3 != sscanf ((char *) s, "%lu/%lu/%lu", &mm, &dd, &yyyy))
            && (3 != sscanf ((char *) s, "%lu-%lu-%lu", &dd, &mm, &yyyy)))
            || (dd > 31)
            || (mm > 12)
            || (yyyy < 1900))
            return -1;

      score_time = (yyyy - 1900) * 10000 + (mm - 1) * 100 + dd;
      return score_time <= today ? 1 : 0;
}

/****
*****
*****  Parsing the scorefile
*****
****/

static void
prep_section_regex (const PString * in, GString * out)
{
      const char * pch;
      const char * end;

      g_string_set_size (out, 0);

      for (pch=in->str, end=pch+in->len; pch!=end; ++pch)
      {
            if ((*pch=='.' || *pch=='+') && (pch==in->str || pch[-1]!='\\'))
                  g_string_append_c (out, '\\');
            else if (*pch=='*' && (pch==in->str || pch[-1]!='\\'))
                  g_string_append_c (out, '.');

            g_string_append_c (out, *pch);
      }
}

static Section*
get_section (const char * name, const char * filename, int line_number)
{
      GSList * l;
      Section * retval = NULL;

      /* try to find a section that already matches that name */
      for (l=_sections; l!=NULL; l=l->next) {
            Section * section = (Section*) l->data;
            if (!pan_strcmp (section->name, name)) {
                  retval = section;
                  break;
            }
      }

      /* if no such section, make one. */
      if (retval == NULL)
      {
            GString * gname;
            GString * prep;
            PString token = PSTRING_INIT;
            const char * march;
            const gboolean negate = *name == '~';

            /* make a new section */
            retval = g_new (Section, 1);
            retval->name = g_string_chunk_insert_const (_string_chunk, name);
            retval->section_name_negate = negate;
            retval->scores = NULL;
            retval->filters = NULL;
            retval->filter_qty = 0;

            /* break the groups apart by whitespace and compile each one as a regular expression */
            if (negate)
                  ++name;

            /* normalize the section name's group delimiters and strip extra spaces */
            gname = g_string_new (name);
            pan_g_string_replace (gname, ",", " "); /* treat commas as spaces */
            pan_g_string_replace (gname, "\t", " "); /* treat tabs as spaces */
            while (strstr (gname->str, "  ")) /* strip out extra spaces */
                  pan_g_string_replace (gname, "  ", " ");

            /* count the number of names */
            march = gname->str;
            while ((get_next_token_pstring (march, ' ', &march, &token)))
                  ++retval->filter_qty;

            /* compile the names into regex_t's */
            prep = g_string_new (NULL);
            if (retval->filter_qty)
            {
                  int real_qty = 0;
                  retval->filters = g_new (FilterPhrase*, retval->filter_qty);
                  march = gname->str;

                  while ((get_next_token_pstring (march, ' ', &march, &token)))
                  {
                        char * errmsg;

                        prep_section_regex (&token, prep);

                        errmsg = filter_phrase_validate_pattern (prep->str);
                        if (errmsg != NULL)
                        {
                              log_add_va (LOG_ERROR, _("Can't use regular expression \"%s\" from file \"%s\", line %d: %s"),
                                          prep->str, filename, line_number, errmsg);
                              g_free (errmsg);
                        }
                        else
                        {
                              FilterPhrase * phrase = FILTER_PHRASE (filter_phrase_new ());

                              filter_phrase_set (phrase,
                                                 PHRASE_MATCH_REGEX,
                                                 PHRASE_KEY_SUBJECT,
                                                 prep->str,
                                                 TRUE);

                              retval->filters[real_qty++] = phrase;
                        }
                  }

                  retval->filter_qty = real_qty;
            }

            /* remember this section */
            _sections = g_slist_append (_sections, retval);

            /* cleanup */
            g_string_free (prep, TRUE);
            g_string_free (gname, TRUE);
      }

      return retval;
}

/**
 * For each line we read that continues a Score entry,
 * update the current score's "end_line" variable.
 */
static void
maybe_update_score_end_line (ScorefileContext * context, int line_number)
{
      g_return_if_fail (context != NULL);

      if (context->current_score)
            context->current_score->end_line = line_number;
}

static int
parse_score_file (ScorefileContext   * context,
                  const char         * filename)
{
      int retval = 0;
      int line_number = 0;
      char * keyword_delimiter = NULL;
      GError * err = NULL;
      GString * str = NULL;
      GIOChannel * in = NULL;

      /* sanity clause */
      g_return_val_if_fail (context!=NULL, -1);
      g_return_val_if_fail (is_nonempty_string (filename), -1);

      /* open the scorefile */
      in = g_io_channel_new_file (filename, "r", &err);
      if (in == NULL) {
            if (err != NULL) {
                  log_add_va (LOG_ERROR, _("Error opening file \"%s\": %s"), filename, err->message);
                  g_error_free (err);
                  err = NULL;
            }
            return -1;
      }

      /* read through the file, one line at a time */
      str = g_string_sized_new (256);
      while (G_IO_STATUS_NORMAL == g_io_channel_read_line_string (in, str, NULL, &err))
      {
            ++line_number;

            /* kill whitespace */
            pan_g_string_strstrip (str);

            /* skip comments & blank lines */
            if (!str->len || *str->str=='%' || *str->str=='#')
                  continue;

            /* new Section */
            if (*str->str=='[')
            {
                  char * name = str->str + 1;
                  char * end = strchr (name, ']');
                  if (end != NULL)
                        *end = '\0';
                  g_strstrip (name);

                  /* get the section */
                  context->current_section = get_section (name, filename, line_number);
                  context->current_score = NULL;
                  g_slist_free (context->aggregates);
                  context->aggregates = NULL;
            }

            /* new Score */
            else if (context->current_section!=NULL
                  && !g_ascii_strncasecmp (str->str, "Score:", 6))
            {
                  char * end;
                  const char * pch;
                  FilterAggregateType aggregate_type;
                  char * score_name = NULL;
                  int value;
                  gboolean value_assign_flag;
                  Score * score;

                  g_string_erase (str, 0, 5);

                  /* how many criteria need to match? */
                  aggregate_type = !g_ascii_strncasecmp (str->str, "::", 2)
                        ? AGGREGATE_TYPE_OR
                        : AGGREGATE_TYPE_AND;

                  /* find the value */
                  pch = str->str;
                  while (*pch==':') ++pch;
                  while (isspace((guchar)*pch)) ++pch;
                  if ((value_assign_flag = (*pch == '='))) {
                        ++pch;
                        while (isspace((guchar)*pch)) ++pch;
                  }
                  value = atoi (pch);

                  /* try to find the score's name */
                  end = strchr (str->str, '#');
                  if (end == NULL)
                        end = strchr (str->str, '%');
                  if (end != NULL) {
                        score_name = end + 1;
                        g_strstrip (score_name);
                        if (!*score_name)
                              score_name = NULL;
                  }

                  /* new score */
                  score = g_new (Score, 1);
                  score->expired = FALSE;
                  score->name = score_name ? g_string_chunk_insert_const (_string_chunk, score_name) : NULL;
                  score->filename = g_string_chunk_insert_const (_string_chunk, filename);
                  score->begin_line = score->end_line = line_number;
                  score->filter = NULL;
                  score->value_assign_flag = value_assign_flag;
                  score->stop_scoring = value_assign_flag;
                  score->value = value;
                  score->filter = filter_aggregate_new ();
                  filter_aggregate_set_type (FILTER_AGGREGATE(score->filter), aggregate_type);
                  g_slist_free (context->aggregates);
                  context->aggregates = g_slist_prepend (NULL, score->filter);
                  context->current_score = score;
                  context->current_section->scores = g_slist_append (context->current_section->scores, score);
            }

            /* Begin Nested conditions */
            else if (*str->str=='{'
                  && str->str[1]==':'
                  && context->aggregates!=NULL)
            {
                  Filter * filter;
                  FilterAggregate * parent = FILTER_AGGREGATE (context->aggregates->data);

                  /* this is part of a score */
                  maybe_update_score_end_line (context, line_number);

                  /* push a new aggregate to the front of the aggregate list */
                  filter = filter_aggregate_new ();
                  filter_aggregate_set_type (FILTER_AGGREGATE(filter), str->str[2]==':'
                        ? AGGREGATE_TYPE_OR
                        : AGGREGATE_TYPE_AND);
                  filter_aggregate_add (FILTER_AGGREGATE(parent), &filter, 1);
                  context->aggregates = g_slist_prepend (context->aggregates, filter);
            }

            /* End Nested conditions */
            else if (*str->str=='}'
                  && g_slist_length(context->aggregates)>1)
            {
                  GSList * l;

                  /* this is part of a score */
                  maybe_update_score_end_line (context, line_number);

                  /* pop the first aggregate out of the aggregate list */
                  l = context->aggregates;
                  context->aggregates = context->aggregates->next;
                  l->next = NULL;
                  g_slist_free_1 (l);
            }

            /* Include another file */
            else if (!g_ascii_strncasecmp (str->str, "include ", 8))
            {
                  char * dirname = g_path_get_dirname (filename);
                  char * include_filename = pan_file_normalize (str->str+8, dirname);
                  int status;

                  /* this can be part of a score */
                  maybe_update_score_end_line (context, line_number);

                  status = parse_score_file (context, include_filename);

                  g_free (include_filename);
                  g_free (dirname);

                  if (status != 0) {
                        retval = status;
                        break;
                  }
            }

            /* Expires */
            else if (context->current_section!=NULL
                  && context->current_section->scores!=NULL
                  && !g_ascii_strncasecmp (str->str, "Expires:", 8))
            {
                  int has_expired;

                  /* this can be part of a score */
                  maybe_update_score_end_line (context, line_number);

                  /* get the date */
                  g_string_erase (str, 0, 8);
                  pan_g_string_strstrip (str);
                  has_expired = has_score_expired (str->str, context->today);
                  if (has_expired < 0) {
                        score_error (_("expecting 'Expires: MM/DD/YYYY' or 'Expires: DD-MM-YYYY'"),
                              str->str, line_number, filename);
                  }
                  else if (has_expired) {
                        score_info (_("expired old score"), str->str, line_number, filename);
                        if (context->current_score != NULL)
                              context->current_score->expired = TRUE;
                  }
            }

            /* new Filter */
            else if (context->aggregates!= NULL
                  && ((keyword_delimiter = strpbrk (str->str, ":="))) != NULL)
            {
                  int key_type = 0;
                  char * key;
                  char * val;
                  gboolean negate = FALSE;
                  gboolean case_sensitive = FALSE;
                  Filter * filter = NULL;

                  /* this is part of a score */
                  maybe_update_score_end_line (context, line_number);

                  /* follow XNews' idiom for specifying
                   * case sensitivity: '=' as the delimiter instead of ':' */
                  case_sensitive = *keyword_delimiter == '=';

                  /* negate? */
                  if (*str->str == '~') {
                        negate = TRUE;
                        g_string_erase (str, 0, 1);
                        --keyword_delimiter;
                  }

                  /* get key/val pairs */
                  key = str->str;
                  *keyword_delimiter = '\0';
                  g_strstrip (key);
                  val = keyword_delimiter + 1;
                  g_strstrip (val);

                  if (!g_ascii_strcasecmp (key, "Lines")) {
                        filter = filter_line_count_new ();
                        FILTER_LINE_COUNT(filter)->minimum_line_count = atoi (val);
                  }
                  else if (!g_ascii_strcasecmp (key, "Bytes")) {
                        filter = filter_bytes_new ();
                        FILTER_BYTES(filter)->minimum_bytes = strtoul (val, NULL, 10);
                  }
                  else if (!g_ascii_strcasecmp (key, "Age")) {
                        filter = filter_date_new ();
                        FILTER_DATE(filter)->minimum_days_old = atoi (val);
                        filter_negate (filter);
                  }
                  else if (!g_ascii_strcasecmp (key, "Subject"))
                        key_type = PHRASE_KEY_SUBJECT;
                  else if (!g_ascii_strcasecmp (key, "From"))
                        key_type = PHRASE_KEY_AUTHOR;
                  else if (!g_ascii_strcasecmp (key, "Message-Id") || !g_ascii_strcasecmp (key, "Message-ID"))
                        key_type = PHRASE_KEY_MESSAGE_ID;
                  else if (!g_ascii_strcasecmp (key, "References"))
                        key_type = PHRASE_KEY_REFERENCES;
                  else if (!g_ascii_strcasecmp (key, "Xref"))
                        key_type = PHRASE_KEY_XREF;
                  else
                        score_error (_("skipping unsupported criteria"), str->str, line_number, filename);
                  if (key_type != 0) {
                        filter = filter_phrase_new ();
                        filter_phrase_set (FILTER_PHRASE(filter), PHRASE_MATCH_REGEX, key_type, val, case_sensitive);
                  }

                  if (filter != NULL)  {
                        FilterAggregate * parent = FILTER_AGGREGATE(context->aggregates->data);
                        if (negate)
                              filter_negate (filter);
                        filter_aggregate_add (parent, &filter, 1);
                        pan_object_unref (PAN_OBJECT(filter));
                  }
            }

            /* Error */
            else {
                  score_error (_("unexpected line."), str->str, line_number, filename);
                  retval = -1;
                  break;
            }
      }
      if (err != NULL) {
            score_error (_("Error reading file: "), err->message, line_number, filename);
            g_error_free (err);
            err = NULL;
            retval = -1;
      }

      g_io_channel_unref (in);
      g_string_free (str, TRUE);
      return retval;
}


#ifdef DEBUG_SCOREFILE
static void
dump_scorefile (void)
{
      GSList * l;
      for (l=_sections; l!=NULL; l=l->next) {
            GSList * l2;
            const Section * section = (const Section*) l->data;
            printf ("\n\nNEW SECTION \"%s\"\n\n",  section->name);
            for (l2=section->scores; l2!=NULL; l2=l2->next) {
                  const Score * score = (const Score *) l2->data;
                  printf ("\t%% location: %s:%d - %d\n", score->filename, score->begin_line, score->end_line);
                  printf ("\tScore: %s %d\n", score->value_assign_flag ? "assign" : "add", score->value);
                  printf ("\t%s\n\n", filter_to_string_deep (score->filter));
            }
      }
      fflush (NULL);
}
#else
#define dump_scorefile()
#endif

static void
remove_unnecessary_filters (void)
{
      GSList * l;
      GSList * l2;

      for (l=_sections; l!=NULL; l=l->next)
      {
            GSList * purge_l;
            GSList * purge_scores = NULL;
            Section * section = (Section*) l->data;

            for (l2=section->scores; l2!=NULL; l2=l2->next)
            {
                  Score * score = (Score*) l2->data;
                  if (filter_isa (score->filter, FILTER_AGGREGATE_CLASS_ID))
                  {
                        FilterAggregate * aggregate = FILTER_AGGREGATE(score->filter);

                        /* only one clause, so we can remove the aggregate */
                        if (aggregate->children->len == 1u)
                        {
                              Filter * filter = FILTER(g_ptr_array_index(aggregate->children,0));
                              pan_object_ref (PAN_OBJECT(filter));
                              pan_object_unref (PAN_OBJECT(aggregate));
                              score->filter = filter;
                              aggregate = NULL;
                        }

                        /* no clauses -- ill-formed scorefile, or unsupported keys removed */
                        else if (aggregate->children->len == 0u)
                        {
                              score_info (_("skipping score because it has no criteria"), score->name, score->begin_line, score->filename);
                              pan_object_unref (PAN_OBJECT(aggregate));
                              g_free (score);
                              l2->data = NULL;
                              purge_scores = g_slist_prepend (purge_scores, l2);
                        }
                  }
            }

            /* remove any scores that were just purged out for having no clauses */
            for (purge_l=purge_scores; purge_l!=NULL; purge_l=purge_l->next)
                  section->scores = g_slist_delete_link (section->scores, purge_l->data);
            g_slist_free (purge_scores);
      }
}

static time_t scorefile_time = (time_t)0;

static int
ensure_scorefile_loaded (void)
{
      int status = 0;
      debug_enter ("ensure_scorefile_loaded");

      if (_string_chunk == NULL)
            _string_chunk = g_string_chunk_new (512);

      if (!scorefile_time)
      {
            int score_size = 0;
            int section_size = 0;
            double diff;
            GTimeVal start;
            GTimeVal finish;
            const char * filename = score_get_main_scorefile_filename ();

            g_get_current_time (&start);

            if (!pan_file_exists (filename))
                  scorefile_time = time (NULL);
            else
            {
                  /* parse the file */
                  ScorefileContext context;
                  context.current_section = NULL;
                  context.current_score = NULL;
                  context.aggregates = NULL;
                  context.today = get_today ();
                  status = parse_score_file (&context, filename);
                  g_slist_free (context.aggregates);
                  remove_unnecessary_filters ();

                  if (!status)
                  {
                        GSList * l;

                        scorefile_time = time (NULL);
                        dump_scorefile ();

                        /* count the number of sections & score for timing stats */
                        for (l=_sections; l!=NULL; l=l->next) {
                              GSList * l2;
                              ++section_size;
                              for (l2=((Section*)(l->data))->scores; l2!=NULL; l2=l2->next)
                                    ++score_size;
                        }
                  }
            }

            g_get_current_time (&finish);
            diff = finish.tv_sec - start.tv_sec;
            diff += (finish.tv_usec - start.tv_usec)/(double)G_USEC_PER_SEC;
            if (score_size != 0)
                  log_add_va (LOG_INFO, _("Loaded %d score entries in %d sections in %.1f seconds (%.0f entries/sec)"),
                              score_size,
                              section_size,
                              diff,
                              score_size/(fabs(diff)<0.001?0.001:diff));
      }

      debug_exit ("ensure_scorefile_loaded");
      return status;
}

static void
unload_scores (void)
{
      GSList * l;
      debug_enter ("unload_scores");

      for (l=_sections; l!=NULL; l=l->next)
      {
            int i;
            GSList * l2;
            Section * section = (Section*) l->data;

            /* free the section's scores */
            for (l2=section->scores; l2!=NULL; l2=l2->next) {
                  Score * score = (Score*) l2->data;
                  pan_object_unref (PAN_OBJECT(score->filter));
                  g_free (score);
            }
            g_slist_free (section->scores);

            /* free the section regex */
            for (i=0; i<section->filter_qty; ++i)
                  pan_object_unref (PAN_OBJECT(section->filters[i]));
            g_free (section->filters);

            /* free the section */
            g_free (section);
      }

      if (_sections != NULL) {
            g_slist_free (_sections);
            _sections = NULL;
      }

      if (_string_chunk != NULL) {
            g_string_chunk_free (_string_chunk);
            _string_chunk = NULL;
      }

      debug_exit ("unload_scores");
}

PanCallback*
score_get_scorefile_invalidated_callback (void)
{
      static PanCallback * cb = NULL;
      if (cb==NULL) cb = pan_callback_new ();
      return cb;
}

static void
fire_scorefile_invalidated (gboolean do_rescore_all)
{
      pan_callback_call (score_get_scorefile_invalidated_callback(), NULL, GINT_TO_POINTER(do_rescore_all));
}

static void
score_invalidate_impl (void)
{
      unload_scores ();

      scorefile_time = 0;
}
void
score_invalidate (gboolean do_rescore_all)
{
      debug_enter ("score_invalidate");

      g_static_mutex_lock (&_score_mutex);
      {
            score_invalidate_impl ();
      }
      g_static_mutex_unlock (&_score_mutex);

      fire_scorefile_invalidated (do_rescore_all);

      debug_exit ("score_invalidate");
}

/****
*****
*****  Adding new scores to the scorefile
*****
****/

static int
append_to_scorefile (const char * str, int str_len)
{
      char * dirname;
      const char * filename;
      GIOStatus status;
      GIOChannel * io;
      GError * err = NULL;
      debug_enter ("append_to_scorefile");

      /* sanity clause */
      g_return_val_if_fail (is_nonempty_string (str), -1);

      /* get the filename and make sure its directory exists */
      filename = score_get_main_scorefile_filename ();
      dirname = g_path_get_dirname (filename);
      pan_file_ensure_path_exists (dirname);
      replace_gstr (&dirname, NULL);

      /* open the scorefile for writnig */
      io = g_io_channel_new_file (filename, "a+", &err);
      if (io == NULL) {
            if (err != NULL) {
                  log_add_va (LOG_ERROR, _("Error opening file \"%s\": %s"), filename, err->message);
                  g_error_free (err);
            }
            return -1;
      }

      status = g_io_channel_write_chars (io, str, str_len, NULL, &err);
      if (status != G_IO_STATUS_NORMAL) {
            if (err != NULL) {
                  log_add_va (LOG_ERROR, _("Error score to file \"%s\": %s"), filename, err->message);
                  g_error_free (err);
            }
            return -1;
      }

      g_io_channel_unref (io);
      debug_exit ("append_to_scorefile");
      return 0;
}

static gboolean
slrn_adds_these_even_if_turned_off (const char * key)
{
      return !g_ascii_strcasecmp (key, "Subject")
          || !g_ascii_strcasecmp (key, "From")
          || !g_ascii_strcasecmp (key, "References")
          || !g_ascii_strcasecmp (key, "Xref");
}

#ifdef G_OS_WIN32
#define EOL "\r\n"
#else
#define EOL "\n"
#endif

static GString*
make_scorefile_entry (const char           * score_name,
                      const char           * section_name,
                      const int              value,
                      const gboolean         value_assign_flag,
                      const int              lifespan_days,
                      FilterAggregateType    item_type,
                      const ScoreAddItem   * items,
                      const int              item_qty)
{
      int i;
      const time_t now = time (NULL);
      GString * str = g_string_sized_new (1024);
      debug_enter ("make_scorefile_entry");
      g_string_append_printf (str, EOL "%%BOS" EOL);
      g_string_append_printf (str, "%%Score created by Pan on %s" EOL, ctime(&now));
      g_string_append_printf (str, "[%s]" EOL,
            (is_nonempty_string(section_name) ? section_name : "*"));
      g_string_append_printf (str, "Score%s %s%d",
            (item_type==AGGREGATE_TYPE_OR ? "::" : ":"),
            (value_assign_flag?"=":""),
            value);
      if (is_nonempty_string (score_name))
            g_string_append_printf (str, " %% %s", score_name);
      g_string_append (str, EOL);
      if (lifespan_days <= 0)
            g_string_append_printf (str, "%%Expires: " EOL);
      else {
            int dd=0, mm=0, yyyy=0;
            time_t expire_time_t = now + lifespan_days * 24 * 3600;
            struct tm expire_tm;
            pan_localtime_r (&expire_time_t, &expire_tm);
            dd = expire_tm.tm_mday;
            mm = expire_tm.tm_mon + 1;
            yyyy = expire_tm.tm_year + 1900;
            g_string_append_printf (str, "Expires: %u/%u/%u" EOL, mm, dd, yyyy);
      }
      for (i=0; i<item_qty; ++i) {
            const ScoreAddItem * item = items + i;
            const char * value = item->value ? item->value : "";
            if (is_nonempty_string(item->value) && (item->on || slrn_adds_these_even_if_turned_off(item->key)))
                  g_string_append_printf (str, "%c\t%s%s: %s" EOL,
                                    (item->on ? ' ' : '%'),
                                    (item->negate ? "~" : ""),
                                    item->key,
                                    value);
      }
      g_string_append_printf (str, "%%EOS" EOL EOL);

      debug_exit ("make_scorefile_entry");
      return str;
}

void 
score_add (const char           * score_name,
           const char           * section_name,
           const int              value,
           const gboolean         value_assign_flag,
           const int              lifespan_days,
           FilterAggregateType    item_type,
           const ScoreAddItem   * items,
           const int              item_qty,
           const gboolean         do_rescore_all)
{
      int i;
      debug_enter ("score_add");

      /* sanity clause */
      g_return_if_fail (value != 0);
      g_return_if_fail (items != NULL);
      g_return_if_fail (item_qty > 0);
      g_return_if_fail (item_type==AGGREGATE_TYPE_AND || item_type==AGGREGATE_TYPE_OR);
      for (i=0; i<item_qty; ++i) {
            g_return_if_fail (is_nonempty_string (items[i].key));
            g_return_if_fail (is_nonempty_string (items[i].value) || !items[i].on);
      }

      /* scorefile is single-threaded */
      g_static_mutex_lock (&_score_mutex);
      {
            GString * str = make_scorefile_entry (score_name, section_name,
                                                  value, value_assign_flag,
                                                  lifespan_days,
                                                  item_type, items, item_qty);

            ensure_scorefile_loaded ();

            /* write the score to the scorefile */
            append_to_scorefile (str->str, str->len);

            /* force any future scoring to reload the scorefile */
            score_invalidate_impl ();

            /* cleanup */
            g_string_free (str, TRUE);
      }
      g_static_mutex_unlock (&_score_mutex);

      fire_scorefile_invalidated (do_rescore_all);

      debug_exit ("score_add");
}

/****
*****
*****  Scoring Articles
*****
****/

typedef void (*ScoreMatchesFunc)(const Section  * section,
                                 const Score    * score,
                                 Article        * article,
                                 gpointer         user_data);

static void
score_article_matchfunc (const Section  * section,
                         const Score    * score,
                         Article        * article,
                         gpointer         user_data)
{
      if (score->value_assign_flag)
            article->score = score->value;
      else
            article->score += score->value;
}

static gboolean
does_section_match (const Section * section, const PString * group_name)
{
      int i;
      gboolean match = FALSE;

      /* sanity clause */
      g_return_val_if_fail (section!=NULL, FALSE);
      g_return_val_if_fail (pstring_is_set (group_name), FALSE);

      /* does it match? */
      for (i=0; !match && i<section->filter_qty; ++i) {
            match = filter_phrase_does_match (section->filters[i], group_name->str, group_name->len);
            if (section->section_name_negate)
                  match = !match;
      }

      return match;
}

static void
score_articles (Group              * group,
                Article           ** articles_in,
                int                  articles_in_qty,
                ScoreMatchesFunc     match_func,
                gpointer             match_func_user_data)
{
      GSList * l;
      GPtrArray * articles;
      GPtrArray * keep_scoring;
      gboolean * match;
      debug_enter ("score_articles");

      articles = g_ptr_array_sized_new (articles_in_qty);
      keep_scoring = g_ptr_array_sized_new (articles_in_qty);
      match = g_new (gboolean, articles_in_qty);

      pan_g_ptr_array_assign (articles, (gpointer*)articles_in, articles_in_qty);

      /* walk through all the sections to see if any apply to this group */
      for (l=_sections; l!=NULL; l=l->next)
      {
            GSList * l2;
            const Section * section = (const Section*) l->data;

            if (!does_section_match (section, &group->name))
                  continue;

            /* test the articles against each score in the section */
            for (l2=section->scores; l2!=NULL; l2=l2->next)
            {
                  guint i;
                  const Score * score = (const Score*) l2->data;

                  if (score->expired)
                        continue;

                  /* prep the arrays */
                  g_ptr_array_set_size (keep_scoring, 0);
                  memset (match, 0, sizeof(gboolean) * articles->len);

                  /* test the articles */
                  filter_test_articles (score->filter, (const Article**)articles->pdata, articles->len, match);

                  /* if they passed the test, call the match func. 
                     if they didn't pass, or the entry doesn't say `stop scoring', Article continues to the next round. */
                  for (i=0u; i!=articles->len; ++i) {
                        Article * article = ARTICLE (g_ptr_array_index (articles, i));
                        if (match[i])
                              (*match_func)(section, score, article, match_func_user_data);
                        if (!match[i] || !score->stop_scoring)
                              g_ptr_array_add (keep_scoring, article);
                  }

                  /* everything that didn't hit 'stop scoring' makes it through to the next round */
                  pan_g_ptr_array_assign (articles, keep_scoring->pdata, keep_scoring->len);
            }
      }

      /* cleanup */
      g_free (match);
      g_ptr_array_free (articles, TRUE);
      g_ptr_array_free (keep_scoring, TRUE);
      debug_exit ("score_articles");
}

void 
ensure_articles_scored  (Group          * group,
                         Article       ** articles_in,
                         int              article_in_qty,
                         GPtrArray      * optional_changed)
{
      debug_enter ("ensure_articles_scored");

      /* sanity clause */
      g_return_if_fail (group_is_valid (group)); 
      g_return_if_fail (article_in_qty >= 0);
      g_return_if_fail (articles_are_valid ((const Article**)articles_in, article_in_qty));
      if (!article_in_qty)
            return;

      g_static_mutex_lock (&_score_mutex);
      {
            if (ensure_scorefile_loaded () >= 0)
            {
                  int i;
                  int article_qty;
                  Article ** articles;

                  /* make a temporary list of articles not scored yet */
                  articles = g_new (Article*, article_in_qty);
                  for (article_qty=i=0; i<article_in_qty; ++i)
                        if (articles_in[i]->score_date < scorefile_time)
                              articles[article_qty++] = articles_in[i];

                  /* score any articles that need it */
                  if (article_qty > 0)
                  {
                        GPtrArray * score_changed;
                        const time_t now = time (NULL);
                        gint16 * old_score;
                        double diff;
                        GTimeVal start;
                        GTimeVal finish;

                        g_get_current_time (&start);

                        /* get the old scores */
                        old_score = g_new (gint16, article_qty);
                        for (i=0; i<article_qty; ++i)
                              old_score[i] = articles[i]->score;

                        /* zero out the scores */
                        for (i=0; i<article_qty; ++i) {
                              articles[i]->score = 0;
                              articles[i]->score_date = now;
                        }

                        /* build the new scores */
                        score_articles (group, articles, article_qty, score_article_matchfunc, NULL);

                        /* post a dirty event for all articles whose scores changed */
                        score_changed = g_ptr_array_sized_new (article_qty);
                        for (i=0; i<article_qty; ++i)
                              if (old_score[i] != articles[i]->score)
                                    g_ptr_array_add (score_changed, articles[i]);
                        if (score_changed->len) {
                              articles_set_dirty ((Article**)score_changed->pdata, score_changed->len);
                              if (optional_changed)
                                    pan_g_ptr_array_append (optional_changed, score_changed->pdata, score_changed->len);
                        }

                        /* cleanup */
                        g_free (old_score);
                        g_ptr_array_free (score_changed, TRUE);

                        /* timing */
                        g_get_current_time (&finish);
                        diff = finish.tv_sec - start.tv_sec;
                        diff += (finish.tv_usec - start.tv_usec)/(double)G_USEC_PER_SEC;
                        log_add_va (LOG_INFO, _("Scored %d entries in %.1f seconds (%.0f articles/sec)"),
                              article_qty,
                              diff,
                              article_qty/(fabs(diff)<0.001?0.001:diff));
                  }

                  /* cleanup */
                  g_free (articles);
            }
      }
      g_static_mutex_unlock (&_score_mutex);

      debug_exit ("ensure_articles_scored");
}

/****
*****
*****  Score Viewing
*****
****/

/**
 * Convenience utility function to free an array of
 * ScoreViewItems returned from score_view_item().
 */
void
score_view_free (GPtrArray * a)
{
      guint i;

      g_return_if_fail (a != NULL);

      for (i=0; i<a->len; ++i)
      {
            ScoreViewItem * s = (ScoreViewItem*) g_ptr_array_index (a, i);
            g_free (s->filename);
            g_free (s->filter_string);
            g_free (s->score_name);
            g_free (s);
      }

      g_ptr_array_free (a, TRUE);
}

/**
 * function invoked by score_articles() via score_view_article(),
 * to add a new ScoreItemView to the array.
 */
static void
view_article_func (const Section * section,
                   const Score    * score,
                   Article        * article,
                   gpointer         user_data)
{
      ScoreViewItem * item;
       
      item = g_new (ScoreViewItem, 1);
      item->filename = g_strdup (score->filename);
      item->begin_line = score->begin_line;
      item->end_line = score->end_line;
      item->filter_string = filter_to_string_deep (score->filter);
      item->score_name = g_strdup (score->name);
      item->value = score->value;
      item->value_assign_flag = score->value_assign_flag;

      g_ptr_array_add ((GPtrArray*)user_data, item);
}

/**
 * Get an array of ScoreViewItems that apply to the user-specified article.
 */
GPtrArray*
score_view_article (const Article * a)
{
      GPtrArray * retval = g_ptr_array_new ();

      /* sanity clause */
      g_return_val_if_fail (article_is_valid (a), retval);
      g_return_val_if_fail (a->group!=NULL, retval);
      g_return_val_if_fail (group_is_valid (a->group), retval);

      /* score the article */
      g_static_mutex_lock (&_score_mutex);
      {
            ensure_scorefile_loaded ();

            score_articles (a->group, (Article**)&a, 1, view_article_func, retval);
      }
      g_static_mutex_unlock (&_score_mutex);

      return retval;
}

/****
*****
*****  Score Removing
*****
****/

/**
 * Remove a score from the scorefile.
 *
 * It's easy to imagine how much damage we could do to a finely-tuned
 * scorefile, so let's comment lines out instead of deleting them.
 */
void
score_remove_score  (const char * filename,
                     const int    begin_line,
                     const int    end_line,
                     gboolean     do_rescore_all)
{
      /* sanity clause */
      g_return_if_fail (is_nonempty_string (filename));
      g_return_if_fail (pan_file_exists (filename));
      g_return_if_fail (begin_line >= 0);
      g_return_if_fail (end_line > begin_line);

      g_static_mutex_lock (&_score_mutex);
      {
            char * filename_new = g_strdup_printf ("%s.new", filename);
            GIOChannel * in = NULL;
            GIOChannel * out = NULL;
            GError * err = NULL;
            gboolean ok = TRUE;

            /* open the current scorefile for reading ... */
            if (ok) {
                  in = g_io_channel_new_file (filename, "r", &err);
                  if (in == NULL) {
                        ok = FALSE;
                        if (err != NULL) {
                              log_add_va (LOG_ERROR, _("Error opening file \"%s\": %s"), filename, err->message);
                              g_error_free (err);
                              err = NULL;
                        }
                  }
            }

            /* open the new scorefile for writing ... */
            if (ok) {
                  out = g_io_channel_new_file (filename_new, "w+", &err);
                  if (out == NULL) {
                        ok = FALSE;
                        if (err != NULL) {
                              log_add_va (LOG_ERROR, _("Error opening file \"%s\": %s"), filename_new, err->message);
                              g_error_free (err);
                              err = NULL;
                        }
                  }
            }

            /* copy the existing scorefile in to the new one,
             * commenting out the specified lines as we go */
            if (ok)
            {
                  int line_number = 0;
                  GString * str = g_string_sized_new (512);
                  for (;;)
                  {
                        GIOStatus status = g_io_channel_read_line_string (in, str, NULL, &err);
                        if (status != G_IO_STATUS_NORMAL)
                              break;

                        ++line_number;

                        /* if we're in the kill range,
                         * and the line isn't already a comment,
                         * then add a comment marker */
                        if (begin_line<=line_number && line_number<=end_line) {
                              char * ch = str->str;
                              while (isspace((guchar)*ch))
                                    ++ch;
                              if (*ch!='%' && *ch!='#')
                                    g_string_insert_c (str, 0, '%');
                        }

                        status = g_io_channel_write_chars (out, str->str, str->len, NULL, &err);
                        if (status != G_IO_STATUS_NORMAL)
                              break;
                  }
                  if (err != NULL) {
                        ok = FALSE;
                        g_message (_("Error removing scorefile entry: %s"), err->message);
                        g_error_free (err);
                        err = NULL;
                  }
                  g_string_free (str, TRUE);
            }

            /* close the files */
            if (in != NULL)
                  g_io_channel_unref (in);
            if (out != NULL)
                  g_io_channel_unref (out);

            /* if all went successfully, bump
             * filename -> filename.bak and
             * filename.new -> filename */
            if (ok) {
                  char * filename_bak = g_strdup_printf ("%s.bak", filename);
                  pan_file_rename (filename, filename_bak);
                  pan_file_rename (filename_new, filename);
                  log_add_va (LOG_INFO,
                            _("Scorefile entry removed -- old scorefile \"%s\" backed up as \"%s\""),
                            filename, filename_bak);
                  g_free (filename_bak);
            }

            /* force any future scoring to reload the scorefile */
            score_invalidate_impl ();

            /* cleanup */
            g_free (filename_new);
      }
      g_static_mutex_unlock (&_score_mutex);

      fire_scorefile_invalidated (do_rescore_all);
}

/***
****
***/

static void
add_group_name_to_section (const PString    * server_name,
                           const PString    * group_name,
                           gulong             number,
                           gpointer           gstring_gpointer)
{
      GString * gstr = (GString*) gstring_gpointer;

      g_string_append_len (gstr, group_name->str, group_name->len);
      g_string_append (gstr, ", ");
}
char*
score_create_section_str (const Article * article)
{
      GString * str;

      /* sanity check */
      g_return_val_if_fail (article_is_valid (article), g_strdup("[*]"));

      /* build the section string */
      str = g_string_new (NULL);
      if (pstring_is_set (&article->xref)) /* add all xrefs */
            xref_header_foreach (article->xref.str, &article->group->server->name, NULL, add_group_name_to_section, str);
      else /* just use the current group */
            add_group_name_to_section (&article->group->server->name, &article->group->name, article->number, str);
      g_string_truncate (str, str->len-2);

      return g_string_free (str, FALSE);
}

Generated by  Doxygen 1.6.0   Back to index