root/feedmelinks/bin/heuristics.sh

#!/bin/sh
# TODO: check for minima before building indices

. env.sh

if [ "$#" -lt 2 ]
then
        echo "usage: heuristics.sh (SPAMMERS | USERS) username [username] [username]..."
        exit 1
fi

if [ "$1" != "SPAMMERS" -a "$1" != "USERS" ]
then
        echo "usage: heuristics.sh (SPAMMERS | USERS) username [username] [username]..."
        exit 2
fi

function output_xml_doc_start () {
        echo "<?xml version='1.0'?>"
        echo "<user_group type='$1'>"
}

function output_xml_doc_end () {
        echo "</user_group>"
}

function output_xml_usr_start () {
        echo "  <user name='$1'>"
        echo "    <heuristics"
}

function output_xml_usr_end () {
        echo "    />"
        echo "  </user>"
}

function output_xml_idx () {
        name=$1
        value=$2
        echo "      $name='$value'"
}

output_xml_doc_start $1
shift
USERS=$*

# db auth:
u=`get_database_user`
db=`get_database_name`
p=`get_database_password`

for USER in $USERS
do

        # number of links to examine:
        scope=10

   double_hyphenated_links=`mysql -u $u -p$p $u -B -e "select count( url ) as 'double_hyphenated_links' from links where userid = '$USER' and url REGEXP '.+-.+-.+\\.com';" | tail -n1`
known_spammer_domain_links=`mysql -u $u -p$p $u -B -e "select count( url ) as 'known_spammer_domain_links' from links where userid = '$USER' and url REGEXP '(ourbigtop|blogspot)\.com';" | tail -n1`
                dot_info_domains=`mysql -u $u -p$p $u -B -e "select count( url ) as 'dot_info_domains' from links where userid = '$USER' and url REGEXP '\\.info(%2F|$)';" | tail -n1`
                   tags_w_commas=`mysql -u $u -p$p $u -B -e "select count( name ) as 'tags_w_commas' from tags where userid = '$USER' and name REGEXP ',';" | tail -n1`
    tags_w_multiple_spaces=`mysql -u $u -p$p $u -B -e "select count( name ) as 'tags_w_multiple_spaces' from tags where userid = '$USER' and name REGEXP '. . ';" | tail -n1`
                     links=`mysql -u $u -p$p $u -B -e "select count( url ) as 'links' from links where userid = '$USER';" | tail -n1`
                            tags=`mysql -u $u -p$p $u -B -e "select count( id ) as 'tags' from tags where userid = '$USER';" | tail -n1`
   last_ten_domains_linked=`mysql -u $u -p$p $u -B -e "select lower(substring_index(url, '.', 2)) from links where userid = '$USER' order by id desc limit ${scope};" | tail -n${scope}`
          has_been_snuffed=`mysql -u $u -p$p $u -B -e "select snuffed from users where userid = '$USER';" | tail -n1`
      # for freshness idx:
                first_link=`mysql -u $u -p$p $u -B -e "select to_days( createDate ) from links where userid = '$USER' ORDER BY id ASC LIMIT 1;" | tail -n1`
                       last_link=`mysql -u $u -p$p $u -B -e "select to_days( createDate ) from links where userid = '$USER' ORDER BY id DESC LIMIT 1;" | tail -n1`

        if [ $links -ne 0 ]
        then

                output_xml_usr_start $USER

                echo "$USER" | grep -q "[0-9][0-9]"
                userid_w_double_digits=$?
                if [ $userid_w_double_digits -eq 1 ]
                then
                        userid_w_double_digits_idx=0
                else
                        userid_w_double_digits_idx=1
                fi
                output_xml_idx "userid_w_double_digits_idx" $userid_w_double_digits_idx

                let unique_domains=`echo $last_ten_domains_linked | xargs -n1  | sort | uniq | wc -l`
                domain_duplicity_idx=`echo "scale=4; 1-($unique_domains/$scope)" | bc`
                output_xml_idx "domain_duplicity_idx" $domain_duplicity_idx
                tags_to_links_idx=`echo "scale = 4; $tags / $links"  | bc`
                output_xml_idx "tags_to_links_idx" $tags_to_links_idx

                double_hyphenated_link_idx=`echo "scale = 4; $double_hyphenated_links / $links" | bc`
                output_xml_idx "double_hyphenated_link_idx" $double_hyphenated_link_idx
                
                dot_info_domain_idx=`echo "scale = 4; $dot_info_domains / $links"  | bc`
                output_xml_idx "dot_info_domain_idx" $dot_info_domain_idx
                
                known_spammer_domain_idx=`echo "scale = 4; $known_spammer_domain_links / $links"  | bc`
                output_xml_idx "known_spammer_domain_idx" $known_spammer_domain_idx

                pheezys_links=11658
                volume_idx=`echo "scale = 4; 1-($links / $pheezys_links)" | bc`
                output_xml_idx "volume_idx" $volume_idx
                
                if [ -z "$has_been_snuffed" -o "$has_been_snuffed" = "NULL" ]
                then
                        has_been_snuffed=0
                fi
                output_xml_idx "has_been_snuffed_idx" $has_been_snuffed

                if [ $tags -ne 0 ]
                then
                        tags_w_commas_idx=`echo "scale = 4; $tags_w_commas / $tags" | bc`
                        output_xml_idx "tags_w_commas_idx" $tags_w_commas_idx

                        tags_w_multiple_spaces_idx=`echo "scale = 4; $tags_w_multiple_spaces / $tags" | bc`
                        output_xml_idx "tags_w_multiple_spaces_idx" $tags_w_multiple_spaces_idx
                fi

                if [ -n "$first_link" -a -n "$last_link" ]
                then
                        let age_in_days=$(($last_link-$first_link))
                        fml_start_days=1799
                        freshness_idx=`echo "scale = 4; 1-($age_in_days / $fml_start_days)" | bc`
                        output_xml_idx "freshness_idx" $freshness_idx
                fi

        output_xml_usr_end
        fi # end if( links )

done

output_xml_doc_end