Changeset 9dc8cfc in adblock2privoxy


Ignore:
Timestamp:
Mar 15, 2014 2:05:53 PM (6 years ago)
Author:
zubr <a@…>
Branches:
master
Children:
7575506
Parents:
b81c15c
Message:

small fixes

Files:
2 deleted
15 edited

Legend:

Unmodified
Added
Removed
  • adblock2privoxy.cabal

    rb81c15c r9dc8cfc  
    11name:           adblock2privoxy 
    22version:        1.0 
    3 cabal-version:  >= 1.2 
     3cabal-version:  >= 1.6 
    44build-type:     Simple 
    5 author:         alexey 
     5author:         Alexey Zubritsky 
     6data-files:      
     7                templates/ab2p.system.action,  
     8                templates/ab2p.system.filter 
     9license:        GPL-3 
     10maintainer:     a@zubr.me 
     11license-file:   LICENSE 
     12stability:      stable 
     13synopsis:       Convert adblock config files to privoxy format 
     14description:     
     15                     Almost all adblock features are supported including 
     16                     - block/unblock requests (on privoxy) 
     17                       all syntax features are supported except for regex templates matching host name 
     18                     - hide/unhide page elements (via CSS) 
     19                       all syntax features are supported 
     20                     - all block request options except for outdated ones: 
     21                       Supported: script, image, stylesheet, object, xmlhttprequest, object-subrequest, subdocument,  
     22                                  document, elemhide, other, popup, third-party, domain=..., match-case, donottrack 
     23                       Unsupported: collapse, background, xbl, ping and dtd 
     24                     .   
     25                     Tested with privoxy version 3.0.21. 
     26                     Element blocking feature requires a webserver to work. See Nginx and Apache config examples provided. 
    627 
    728executable adblock2privoxy 
     
    2445                   InputParser, 
    2546                   ParsecExt, 
    26                    Foo, 
    2747                   Utils, 
    2848                   ParserExtTests, 
  • notes.txt

    rb81c15c r9dc8cfc  
     1ошибки: 
     2http://docs.angularjs.org/api/ng.$http 
     3speedtest.net 
     4 
    15еще: 
    26поделить цсс на части, чтоб хром ел 
  • src/ElementBlocker.hs

    rb81c15c r9dc8cfc  
    1010import Utils 
    1111import System.IO hiding (hGetContents) 
    12 import System.FilePath.Posix 
     12import System.FilePath 
    1313import Data.List  
    1414import System.Directory 
    1515import qualified Templates  
    1616import Control.Monad  
    17 import Debug.Trace (traceShow) 
    1817   
    1918 
     
    3837        in do  
    3938           isDir <- doesDirectoryExist entry 
    40            case isDir of 
    41                 True -> when (head entry' /= '.') $ removeDirectoryRecursive entry     
    42                 False -> when (takeExtension entry == ".css") $ removeFile entry              
     39           if isDir then when (head entry' /= '.') $ removeDirectoryRecursive entry     
     40                    else when (takeExtension entry == ".css") $ removeFile entry              
    4341    writeBlockTree :: String -> String -> BlockedRulesTree -> IO () 
    4442    writeBlockTree nodePath debugNodePath (Node name patterns children) = 
     
    6967         do outFile <- openFile filename WriteMode 
    7068            hPutStrLn outFile "/*" 
    71             _ <- mapM (hPutStrLn outFile) $ info 
     69            _ <- mapM (hPutStrLn outFile) info 
    7270            hPutStrLn outFile "*/" 
    7371            hPutStrLn outFile content 
  • src/InputParser.hs

    rb81c15c r9dc8cfc  
    2121import Control.Monad 
    2222import Text.Parsec.Permutation 
    23 import System.FilePath.Posix 
    24 import Utils 
     23import System.FilePath 
    2524  
    2625-------------------------------------------------------------------------- 
  • src/Main.hs

    rb81c15c r9dc8cfc  
    99import SourceInfo 
    1010import System.Console.GetOpt 
    11 import Data.Maybe  
    1211import System.Environment 
    1312import Control.Monad 
    14  
    15 f1, f2, f3, f4, f5, outDir, baseDir :: String 
    16 baseDir = "/home/alexey/Projects/AdBlock2Privoxy/test-data/" 
    17 f1 = baseDir ++ "easylist.txt" 
    18 f2 = baseDir ++ "advblock.txt" -- Ru Ad  
    19 f3 = baseDir ++ "bitblock.txt" -- Ru trash blocks includes fanboy-annoyance 
    20 f4 = baseDir ++ "easyprivacy.txt" 
    21 f5 = baseDir ++ "my.txt" 
    22 outDir = "/home/alexey/test/ab" 
    23 --filenames :: [String] 
    24 --filenames = [f1, f2, f3, f4, f5] 
    25 --filenames = ["/home/alexey/Projects/adblock2privoxy/test-data/testData"] 
    26  
     13import Templates (writeTemplateFiles) 
     14import Data.Time.Clock (getCurrentTime) 
    2715 
    2816data Options = Options 
     
    3927     , Option ['p']     ["privoxyDir"] 
    4028         (ReqArg (\ f opts -> opts { _privoxyDir = f }) 
    41                  "FILE") 
    42          "output FILE" 
     29                 "PATH") 
     30         "privoxy config output path" 
    4331     , Option ['w']     ["webDir"] 
    4432         (ReqArg (\ f opts -> opts { _webDir = f }) 
    45                  "FILE") 
    46          "input FILE" 
     33                 "PATH") 
     34         "css files output path" 
    4735     ] 
    4836 
     
    5846  where  
    5947        writeError msg = ioError $ userError $ msg ++ usageInfo header options 
    60         header = "Usage: adblock2privoxy [OPTION...] files..." 
     48        header = "Usage: adblock2privoxy [OPTION...] adblockFiles..." 
    6149 
    6250 
     
    7159                Left msg -> return ([], NoInfo, inputFile) <$ putStrLn $ show msg 
    7260                     
    73         (parsed, sourceInfo, handlers) <- unzip3 <$> mapM parseFile filenames    
     61        (parsed, sourceInfo, handlers) <- unzip3 <$> mapM parseFile filenames 
     62        showInfo' <- showInfo <$> getCurrentTime     
    7463        let parsed' = concat parsed  
    75             info    = (sourceInfo >>= showInfo) ++ ["------- end ------\n"]                
     64            info    = (sourceInfo >>= showInfo') ++ ["------- end ------\n"]                
    7665        stat privoxyDir info parsed' 
    7766        elemBlock webDir info parsed' 
    7867        urlBlock privoxyDir info parsed' 
     68        writeTemplateFiles privoxyDir 
    7969        sequence $ hClose <$> handlers 
    8070         
  • src/ParsecExt.hs

    rf7023f1 r9dc8cfc  
    11{-# LANGUAGE RankNTypes, ScopedTypeVariables, FlexibleInstances #-} 
    22module ParsecExt ( 
    3     testParsecExt, 
    4     testParseMorse, 
    53    CasesParser, 
    64    StateParser, 
     
    1917import Control.Monad.State 
    2018import Data.Maybe 
    21 import  {-# SOURCE #-} ParserExtTests  
    2219         
    23 --------------------------------------------------------------------------------------------- 
    24 ------------------------- usage samples ------------------------------------------------------ 
    25 --------------------------------------------------------------------------------------------- 
    26  
    27 type ExampleCase = ([String], String, String) 
    28  
    29 parsersChain :: [StringStateParser ExampleCase] 
    30 parsersChain = square3 prefix mid suffix 
    31         where -- all parsers except for last one should consume some input and give some output 
    32             prefix = manyCases ((:[]) <$> (string "ab" <|> string "zz")) 
    33             mid =    many1Cases $ (:[]) <$> letter            -- list of letters 
    34             suffix = many1Cases $ try $ many1 alphaNum   
    35              
    36  
    37 testParsecExt :: Either ParseError [([String], String, String)] 
    38 testParsecExt =  parse (cases parsersChain <* string "$$") "x" "abebz12$$" 
    39  
    40 testParseMorse :: Either ParseError [String] 
    41 testParseMorse = parseMorse "......-...-..---" 
    42  
    43 --------------------------------------------------------------------------------------------- 
    44 --------------------------------------------------------------------------------------------- 
    45 --------------------------------------------------------------------------------------------- 
    46  
    4720-- parser should consume some input to prevent infinite loop 
    4821manyCases :: (Monoid a, Monoid st) => Parser a -> StateParser st a 
  • src/ParserExtTests.hs

    r861fbae r9dc8cfc  
    1111import Data.Maybe 
    1212import Control.Monad.State 
     13 
     14--------------------------------------------------------------------------------------------- 
     15------------------------- parsec ext usage samples ------------------------------------------ 
     16--------------------------------------------------------------------------------------------- 
     17 
     18type ExampleCase = ([String], String, String) 
     19 
     20parsersChain :: [StringStateParser ExampleCase] 
     21parsersChain = square3 prefix mid suffix 
     22        where -- all parsers except for last one should consume some input and give some output 
     23            prefix = manyCases ((:[]) <$> (string "ab" <|> string "zz")) 
     24            mid =    many1Cases $ (:[]) <$> letter            -- list of letters 
     25            suffix = many1Cases $ try $ many1 alphaNum   
     26             
     27 
     28testParsecExt :: Either ParseError [([String], String, String)] 
     29testParsecExt =  parse (cases parsersChain <* string "$$") "x" "abebz12$$" 
     30 
     31testParseMorse :: Either ParseError [String] 
     32testParseMorse = parseMorse "......-...-..---" 
    1333 
    1434-------------------------------------------------------------------------------- 
     
    95115 
    96116morseParsers :: [StringStateParser (ZipListM String)] 
    97 morseParsers = (repeat morseParser) <*> [0..] 
     117morseParsers = repeat morseParser <*> [0 ..] 
    98118 
    99119parseMorse :: String -> Either ParseError [String] 
    100 parseMorse s = fmap (filter $ isPrefixOf "HELL") $ (fmap.fmap) postProcess $ parseMorseRaw "x" s 
     120parseMorse s = (fmap.fmap) postProcess $ parseMorseRaw "x" s 
    101121            where  
    102122            parseMorseRaw =  parse (cases $ morseParsers)  
  • src/SourceInfo.hs

    rb81c15c r9dc8cfc  
    1616 
    1717data SourceInfo = SourceInfo { _title, _filename, _license, _homepage :: String,  
    18                                _lastUpdated :: UTCTime, _expires, _version :: Int } | NoInfo 
     18                               _lastUpdated :: UTCTime, _expires, _version :: Integer } | NoInfo 
    1919 
    2020emptySourceInfo :: SourceInfo 
    2121emptySourceInfo = SourceInfo "" "" "" "" (UTCTime (ModifiedJulianDay 0) (secondsToDiffTime 0) ) 72 0 
    2222 
    23 showInfo :: SourceInfo -> [String]  
    24 showInfo NoInfo = ["----- a source skipped -----"] 
    25 showInfo sourceInfo@(SourceInfo _ filename _ _ lastUpdated expires _) =  
     23showInfo :: UTCTime -> SourceInfo -> [String]  
     24showInfo _ NoInfo = ["----- a source skipped -----"] 
     25showInfo now sourceInfo@(SourceInfo _ filename _ _ lastUpdated expires _) =  
    2626                    [concat ["----- source -----"]] 
    2727                    ++ optionalLine "Title: " _title 
    2828                    ++ [concat ["Filename: ", filename], 
    2929                    concat ["Last modified: ", formatTime defaultTimeLocale "%d %b %Y %H:%M %Z" lastUpdated], 
    30                     concat ["Expires: ", show expires, " hours"]]  
     30                    concat ["Expires: ", show expires, " hours", expired]]  
    3131                    ++ optionalLine "Version: " _version 
    3232                    ++ optionalLine "License: " _license 
    3333                    ++ optionalLine "Homepage: " _homepage 
    34     where optionalLine caption getter | getter sourceInfo == getter emptySourceInfo = [] 
    35                                       | otherwise       = [concat [caption, show $ getter sourceInfo]]  
     34    where  
     35    expired | (diffUTCTime now lastUpdated) > (fromInteger $ expires * 60 * 60) = " (expired)" 
     36                                | otherwise = [] 
     37    optionalLine caption getter | getter sourceInfo == getter emptySourceInfo = [] 
     38                                | otherwise = [concat [caption, show $ getter sourceInfo]]  
    3639 
    3740extractInfo :: [Line] -> SourceInfo 
  • src/Statistics.hs

    rb81c15c r9dc8cfc  
    44import Data.Maybe  
    55import System.IO 
    6 import System.FilePath.Posix 
     6import System.FilePath 
    77import Control.Applicative ((<$>)) 
     8import Control.Monad.State 
    89 
    910type Stat = Map.Map String Int  
     
    1213stat path info lns =  
    1314    let result = collectStat lns  
    14         filename = path </> "stat.txt" 
     15        filename = path </> "ab2p.stat" 
    1516        resultLine (name, value) = concat [name, ": ", show value]  
    1617        errorLine (Line position (Error text))  
     
    2526 
    2627collectStat :: [Line] -> Stat 
    27 collectStat = foldr getStat (Map.empty) 
     28collectStat = foldr getStat Map.empty 
    2829 
    2930increment :: String -> Stat-> Stat 
    30 increment key statData = Map.insertWith (+) key 1 statData 
     31increment key = Map.insertWith (+) key 1 
    3132 
    3233isJustFilled :: Maybe [a] -> Bool 
     
    3435isJustFilled (Just list) = not.null $ list 
    3536 
     37 
    3638getStat :: Line -> Stat-> Stat 
    37 getStat  (Line _ Comment {} ) = increment "comments" 
    38 getStat  (Line _ Error {}) = increment "errors" 
    39 getStat  (Line _ ElementHide {}) = increment "elemHide" 
    40 getStat  (Line _ (RequestBlock policy _ (RequestOptions requestType thirdParty domains _ _ _ _ _))) = r 
     39getStat  (Line _ Comment {} ) = increment "Comments" 
     40getStat  (Line _ Error {}) = increment "Errors" 
     41getStat  (Line _ ElementHide {}) = increment "Elements hiding rules" 
     42getStat  (Line _ (RequestBlock policy _ (RequestOptions _ thirdParty domains _ _ _ _ _))) = execState stateState 
    4143    where  
    42     r s = r8 s 
    43     r1 = increment "RBlock" 
    44     r2 | (policy == InputParser.Unblock) = r1 . increment "excludeRBlock" 
    45        | otherwise = r1 
    46     r3 | isJust thirdParty = r2 . increment "thidrPartyRBlock" 
    47        | otherwise = r2 
    48     r4 | (not.null._negative $ domains) || ((isJustFilled . _positive) $ domains) = r3 . increment "domainsRBlock" 
    49        | otherwise = r3 
    50     r5 | isJust thirdParty && ((not.null._negative $ domains) || ((isJustFilled . _positive) $ domains)) = r4 . increment "domains&thirdPartyRBlock" 
    51        | otherwise = r4 
    52     r6 | ((not.null._negative $ requestType) && ((isJustFilled . _positive) $ requestType)) = r5 . increment "mixedRequestTypeRBlock" 
    53        | otherwise = r5 
    54     r7 | ((not.null._negative $ requestType) || ((isJustFilled . _positive) $ requestType)) = r6 . increment "requestTypeRBlock" 
    55        | otherwise = r6 
    56     r8 | isJust thirdParty && ((not.null._negative $ requestType) || ((isJustFilled . _positive) $ requestType)) = r7 . increment "requestType&thirdPartyRBlock" 
    57        | otherwise = r7 
    58         
    59 --TODO: Elemhide only in unblock requests, only in positive part 
    60 --Document only in unblock requests  
     44    incrementState = modify . increment 
     45    stateState = do 
     46        incrementState "Request block rules total" 
     47        when (policy == InputParser.Unblock) $ incrementState "Request block rules for exception" 
     48        when (isJust thirdParty) $ incrementState "Rules with third party option" 
     49        when ((not.null._negative $ domains) || (isJustFilled . _positive $ domains)) $ incrementState "Request block rules with domain option" 
     50        when ((not.null._negative $ domains) || (isJustFilled . _positive $ domains)) $ incrementState "Request block rules with request type options" 
     51         
     52 
    6153        
    6254        
    6355        
    64         
  • src/Templates.hs

    rb81c15c r9dc8cfc  
    11module Templates where 
    22import  {-# SOURCE #-}  UrlBlocker 
     3import Paths_adblock2privoxy 
     4import System.Directory (copyFile) 
     5import System.FilePath ((</>)) 
    36 
    47blockCss, ab2pPrefix, actionsFilePrefix, filtersFilePrefix :: String 
     
    1114terminalActionSwitch True Request =  
    1215 "+block{ adblock rules } \\\n\ 
    13  \+server-header-tagger{ab2p-block-s} \\\n\ 
    14  \+client-header-tagger{ab2p-handle-as-image-c} \\\n\  
    15  \+server-header-tagger{ab2p-handle-as-image-s}" 
     16 \+server-header-tagger{ab2p-block-s}" 
    1617terminalActionSwitch False Request =  
    1718 "-block \\\n\ 
     
    2728terminalActionSwitch True Dnt = "+add-header{DNT: 1}" 
    2829terminalActionSwitch _ _ = ""  
    29   
     30 
     31writeTemplateFiles :: String -> IO () 
     32writeTemplateFiles outDir = do 
     33        copySystem "ab2p.system.action" 
     34        copySystem "ab2p.system.filter" 
     35        where  
     36        copySystem file = do 
     37                dataDir <- getDataDir 
     38                copyFile (dataDir  </> "templates" </> file) (outDir </> file) 
  • src/UrlBlocker.hs

    rb81c15c r9dc8cfc  
    1818import Data.Maybe    
    1919import System.IO   
    20 import System.FilePath.Posix 
     20import System.FilePath 
    2121import PatternConverter           
    2222 
  • templates/ab2p.system.action

    rb81c15c r9dc8cfc  
    33{+set-image-blocker{blank} \ 
    44 +filter{ab2p-elemhide-filter} \ 
    5  +client-header-tagger{ab2p-elemhide-check-debug}} 
     5 +client-header-tagger{ab2p-elemhide-check-debug} \ 
     6 +client-header-tagger{ab2p-handle-as-image-c} \ 
     7 +server-header-tagger{ab2p-handle-as-image-s}} 
    68/ 
    79 
     
    2426{-block} 
    2527TAG:^ab2p-unblock-s$ 
     28 
     29#workaround for gzip handling privoxy bugs 
     30{ +prevent-compression } 
     31 *.yahoo.com 
     32 *.amazon.com 
  • templates/ab2p.system.filter

    rb81c15c r9dc8cfc  
    1717 
    1818SERVER-HEADER-TAGGER: ab2p-handle-as-image-s 
    19 s@^content-type(?=:.*(?:image\/)):.*@ab2p-handle-as-image@Ti 
     19s@^content-type(?=:[\s\w]*image\/):.*@ab2p-handle-as-image@Ti 
    2020 
    2121CLIENT-HEADER-TAGGER: ab2p-handle-as-image-c 
    22 s@^accept(?=:(?:image\/)):.*@ab2p-handle-as-image@Ti 
     22s@^accept(?=:[\s\w]*image\/):.*@ab2p-handle-as-image@Ti 
    2323 
    2424SERVER-HEADER-FILTER: ab2p-xframe-filter 
     
    2929 
    3030FILTER: ab2p-elemhide-filter 
    31 s@[^'"]\s*<head.*>(?=\s*[^'"])@$&<link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/ab2p.common.css"></link><link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/$host/ab2p.css"></link>@isUD 
     31s@[^'"\s]\s*<head[^>]*>(?=\s*[^'"\s])@$&<link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/ab2p.common.css"></link><link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/$host/ab2p.css"></link>@iUD 
    3232 
    3333FILTER: ab2p-elemhide-filter-debug 
    34 s@[^'"]\s*<head.*>(?=\s*[^'"])@$&<link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/debug/ab2p.common.css"></link><link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/$host.debug/ab2p.css"></link>@isUD 
     34s@[^'"\s]\s*<head[^>]*>(?=\s*[^'"\s])@$&<link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/debug/ab2p.common.css"></link><link rel="stylesheet" type="text/css" href="http://privoxy.zubr.me/$host.debug/ab2p.css"></link>@iUD 
    3535 
    3636FILTER: ab2p-popup-filter 
    37 s@[^'"]\s*<head.*>(?=\s*[^'"])@$&<script type="text/javascript">parent==window&&opener&&close()</script>@isUD 
     37s@[^'"\s]\s*<head[^>]*>(?=\s*[^'"\s])@$&<script type="text/javascript">parent==window&&opener&&close()</script>@iUD 
    3838 
    3939 
  • test-data/my.txt

    rb81c15c r9dc8cfc  
    33|r.e1.ru 
    44||yahoo.com$image,popup 
     5@@/tracker.php?$document 
     6@@||sourceforge.net/tracker/$document 
     7||adlift.ru/  
    58 
  • test-data/testData

    rb81c15c r9dc8cfc  
    88!||adm.fwmrm.net/p/msnbc_live$object-subrequest,third-party,domain=~msnbc.msn.com,~www.nbcnews.com 
    99!/mydirtyhobby$domain=mydirtyhobby.com,mydirtyhobby.de 
    10 yandex.ru$third-party 
    11 se7en.ru##div.sign > a[target="_blank"] 
    12 |r.e1.ru 
    13 ||yahoo.com$image,popup 
    14 ||some.test*net 
    15 super.us$popup,~popup,third-party 
     10://counter. 
     11@@/tracker.php?$document 
     12||opa.na$document 
Note: See TracChangeset for help on using the changeset viewer.