// (C) 2004-2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany

// first published 2004 on http://yacy.net

//

// This is a part of YaCy, a peer-to-peer based web search engine

//

// $LastChangedDate$

// $LastChangedRevision$

// $LastChangedBy$

//

// LICENSE

//

// This program is free software; you can redistribute it and/or modify

// it under the terms of the GNU General Public License as published by

// the Free Software Foundation; either version 2 of the License, or

// (at your option) any later version.

//

// This program is distributed in the hope that it will be useful,

// but WITHOUT ANY WARRANTY; without even the implied warranty of

// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

// GNU General Public License for more details.

//

// You should have received a copy of the GNU General Public License

// along with this program; if not, write to the Free Software

// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

   This class holds the run-time environment of the plasma

   Search Engine. It's data forms a blackboard which can be used

   to organize running jobs around the indexing algorithm.

   The blackboard consist of the following entities:

   - storage: one plasmaStore object with the url-based database

   - configuration: initialized by properties once, then by external functions

   - job queues: for parsing, condensing, indexing

*/

    // load slots

    public  static int  xstackCrawlSlots     = 2000;

    public  static long lastPPMUpdate        = System.currentTimeMillis()- 30000;

    private static final int  dhtMaxContainerCount = 500;

    private int  dhtMaxReferenceCount = 1000;

    // colored list management

    public static SortedSet<String> badwords       = new TreeSet<String>(NaturalOrder.naturalComparator);

    public static SortedSet<String> stopwords      = new TreeSet<String>(NaturalOrder.naturalComparator);

    public static SortedSet<String> blueList       = null;

    public static HandleSet badwordHashes  = null;

    public static HandleSet blueListHashes = null;

    public static HandleSet stopwordHashes = null;

    public static Blacklist urlBlacklist = null;

    public static WikiParser wikiParser = null;

    // storage management

    public  File                           htCachePath;

    public final File                      dictionariesPath;

    public  File                           listsPath;

    public  File                           htDocsPath;

    public  File                           workPath;

    public  File                           releasePath;

    public  File                           networkRoot;

    public  File                           queuesRoot;

    public  File                           surrogatesInPath;

    public  File                           surrogatesOutPath;

    public  Segments                       indexSegments;

    public  LoaderDispatcher               loader;

    public  CrawlSwitchboard               crawler;

    public  CrawlQueues                    crawlQueues;

    public  CrawlStacker                   crawlStacker;

    public  MessageBoard                   messageDB;

    public  WikiBoard                      wikiDB;

    public  BlogBoard                      blogDB;

    public  BlogBoardComments              blogCommentDB;

    public  RobotsTxt                      robots;

    public  Map<String, Object[]>          outgoingCookies, incomingCookies;

    public  volatile long                  proxyLastAccess, localSearchLastAccess, remoteSearchLastAccess;

    public  Network                        yc;

    public  ResourceObserver               observer;

    public  UserDB                         userDB;

    public  BookmarksDB                    bookmarksDB;

    public  WebStructureGraph              webStructure;

    public  ConcurrentHashMap<String, TreeSet<Long>> localSearchTracker, remoteSearchTracker; // mappings from requesting host to a TreeSet of Long(access time)

    public  long                           indexedPages = 0;

    public  int                            searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used

    public  int                            searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication

    public  float                          searchQueriesGlobal = 0f; // partial counter of remote queries (1/number-of-requested-peers)

    public  SortedMap<byte[], String>      clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used

    public  URLLicense                     licensedURLs;

    public  List<Pattern>                  networkWhitelist, networkBlacklist;

    public  FilterEngine                   domainList;

    private Dispatcher                     dhtDispatcher;

    public  LinkedBlockingQueue<String>    trail;

    public  SeedDB                         peers;

    public  WorkTables                     tables;

    public WorkflowProcessor<indexingQueueEntry> indexingDocumentProcessor;

    public WorkflowProcessor<indexingQueueEntry> indexingCondensementProcessor;

    public WorkflowProcessor<indexingQueueEntry> indexingAnalysisProcessor;

    public WorkflowProcessor<indexingQueueEntry> indexingStorageProcessor;

    public RobotsTxtConfig robotstxtConfig = null;

    public boolean useTailCache;

    public boolean exceed134217727;

    private final Semaphore shutdownSync = new Semaphore(0);

    private boolean terminate = false;

    //private Object  crawlingPausedSync = new Object();

    //private boolean crawlingIsPaused = false;

    public Hashtable<String, Object[]> crawlJobsStatus = new Hashtable<String, Object[]>();

    private static Switchboard sb = null;

    public Switchboard(final File dataPath, final File appPath, final String initPath, final String configPath) throws IOException {

        super(dataPath, appPath, initPath, configPath);

        // check if port is already occupied

        final int port = getConfigInt("port", 8090);

        try {

            if (TimeoutRequest.ping("127.0.0.1", port, 500)) {

                throw new RuntimeException("a server is already running on the YaCy port " + port + "; possibly another YaCy process has not terminated yet. Please stop YaCy before running a new instance.");

            }

        } catch (final ExecutionException e1) {

        }

        MemoryTracker.startSystemProfiling();

        sb = this;

        // set loglevel and log

        setLog(new Log("YACY_SEARCH"));

        // set default peer name

        Seed.ANON_PREFIX = getConfig("peernameprefix", "_anon");

        // UPnP port mapping

        if (getConfigBool(SwitchboardConstants.UPNP_ENABLED, false)) {

            InstantBusyThread.oneTimeJob(UPnP.class, "addPortMapping", UPnP.log, 0);

        }

        // init TrayIcon if possible

        Tray.init(this);

        // remote proxy configuration

        initRemoteProxy();

        // memory configuration

        this.useTailCache = getConfigBool("ramcopy", true);

        if (MemoryControl.available() > 1024 * 1024 * 1024 * 1) {

            this.useTailCache = true;

        }

        this.exceed134217727 = getConfigBool("exceed134217727", true);

        if (MemoryControl.available() > 1024 * 1024 * 1024 * 2) {

            this.exceed134217727 = true;

        }

        // load values from configs

        final File indexPath = getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, SwitchboardConstants.INDEX_PATH_DEFAULT);

        this.log.logConfig("Index Primary Path: " + indexPath.toString());

        this.listsPath      = getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);

        this.log.logConfig("Lists Path:     " + this.listsPath.toString());

        this.htDocsPath   = getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);

        this.log.logConfig("HTDOCS Path:    " + this.htDocsPath.toString());

        this.workPath   = getDataPath(SwitchboardConstants.WORK_PATH, SwitchboardConstants.WORK_PATH_DEFAULT);

        this.log.logConfig("Work Path:    " + this.workPath.toString());

        this.dictionariesPath = getDataPath(SwitchboardConstants.DICTIONARY_SOURCE_PATH, SwitchboardConstants.DICTIONARY_SOURCE_PATH_DEFAULT);

        this.log.logConfig("Dictionaries Path:" + this.dictionariesPath.toString());

        // init global host name cache

        this.workPath.mkdirs();

        Domains.init(new File(this.workPath, "globalhosts.list"));

        // init sessionid name file

        final String sessionidNamesFile = getConfig("sessionidNamesFile","defaults/sessionid.names");

        this.log.logConfig("Loading sessionid file " + sessionidNamesFile);

        MultiProtocolURI.initSessionIDNames(FileUtils.loadList(new File(getAppPath(), sessionidNamesFile)));

        // init tables

        this.tables = new WorkTables(this.workPath);

        // init libraries

        this.log.logConfig("initializing libraries");

        new Thread() {

            @Override

            public void run() {

                LibraryProvider.initialize(Switchboard.this.dictionariesPath);

            }

        }.start();

        // set a high maximum cache size to current size; this is adopted later automatically

        final int wordCacheMaxCount = (int) getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);

        setConfig(SwitchboardConstants.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));

        // load the network definition

        overwriteNetworkDefinition();

        // start indexing management

        this.log.logConfig("Starting Indexing Management");

        final String networkName = getConfig(SwitchboardConstants.NETWORK_NAME, "");

        final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong("filesize.max.other", Integer.MAX_VALUE);

        final int redundancy = (int) sb.getConfigLong("network.unit.dhtredundancy.senior", 1);

        final int partitionExponent = (int) sb.getConfigLong("network.unit.dht.partitionExponent", 0);

        this.networkRoot = new File(new File(indexPath, networkName), "NETWORK");

        this.queuesRoot = new File(new File(indexPath, networkName), "QUEUES");

        this.networkRoot.mkdirs();

        this.queuesRoot.mkdirs();

        final File mySeedFile = new File(this.networkRoot, SeedDB.DBFILE_OWN_SEED);

        this.peers = new SeedDB(

                this.networkRoot,

                "seed.new.heap",

                "seed.old.heap",

                "seed.pot.heap",

                mySeedFile,

                redundancy,

                partitionExponent,

                this.useTailCache,

                this.exceed134217727);

        // initialize index

        ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);

        final File oldSingleSegment = new File(new File(indexPath, networkName), "TEXT");

        final File newSegmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");

        Segments.migrateOld(oldSingleSegment, newSegmentsPath, getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default"));

        this.indexSegments = new Segments(

                this.log,

                newSegmentsPath,

                wordCacheMaxCount,

                fileSizeMax,

                this.useTailCache,

                this.exceed134217727);

        // set the default segment names

        setDefaultSegments();

        // load domainList

        try {

          this.domainList = null;

          if(!getConfig("network.unit.domainlist", "").equals("")) {

            final Reader r = getConfigFileFromWebOrLocally(getConfig("network.unit.domainlist", ""), getAppPath().getAbsolutePath(), new File(this.networkRoot, "domainlist.txt"));

            this.domainList = new FilterEngine();

            this.domainList.loadList(new BufferedReader(r), null);

          }

    } catch (final FileNotFoundException e) {

      this.log.logSevere("CONFIG: domainlist not found: " + e.getMessage());

    } catch (final IOException e) {

      this.log.logSevere("CONFIG: error while retrieving domainlist: " + e.getMessage());

    }

        // create a crawler

        this.crawler = new CrawlSwitchboard(

                networkName,

                this.log,

                this.queuesRoot);

        // start yacy core

        this.log.logConfig("Starting YaCy Protocol Core");

        this.yc = new Network(this);

        InstantBusyThread.oneTimeJob(this, "loadSeedLists", Network.log, 0);

        //final long startedSeedListAquisition = System.currentTimeMillis();

        // init a DHT transmission dispatcher

        this.dhtDispatcher = (this.peers.sizeConnected() == 0) ? null : new Dispatcher(

                this.indexSegments.segment(Segments.Process.LOCALCRAWLING),

                this.peers,

                true,

                10000);

        // set up local robots.txt

        this.robotstxtConfig = RobotsTxtConfig.init(this);

        // setting timestamp of last proxy access

        this.proxyLastAccess = System.currentTimeMillis() - 10000;

        this.localSearchLastAccess = System.currentTimeMillis() - 10000;

        this.remoteSearchLastAccess = System.currentTimeMillis() - 10000;

        this.webStructure = new WebStructureGraph(new File(this.queuesRoot, "webStructure.map"));

        // configuring list path

        if (!(this.listsPath.exists())) {

            this.listsPath.mkdirs();

        }

        // load coloured lists

        if (blueList == null) {

            // read only once upon first instantiation of this class

            final String f = getConfig(SwitchboardConstants.LIST_BLUE, SwitchboardConstants.LIST_BLUE_DEFAULT);

            final File plasmaBlueListFile = new File(f);

            if (f != null) blueList = SetTools.loadList(plasmaBlueListFile, NaturalOrder.naturalComparator); else blueList= new TreeSet<String>();

            blueListHashes = Word.words2hashesHandles(blueList);

            this.log.logConfig("loaded blue-list from file " + plasmaBlueListFile.getName() + ", " +

            blueList.size() + " entries, " +

            ppRamString(plasmaBlueListFile.length()/1024));

        }

        // load blacklist

        this.log.logConfig("Loading blacklist ...");

        final File blacklistsPath = getDataPath(SwitchboardConstants.LISTS_PATH, SwitchboardConstants.LISTS_PATH_DEFAULT);

        urlBlacklist = new Blacklist(blacklistsPath);

        ListManager.switchboard = this;

        ListManager.listsPath = blacklistsPath;

        ListManager.reloadBlacklists();

        // load badwords (to filter the topwords)

        if (badwords == null || badwords.isEmpty()) {

            final File badwordsFile = new File(appPath, SwitchboardConstants.LIST_BADWORDS_DEFAULT);

            badwords = SetTools.loadList(badwordsFile, NaturalOrder.naturalComparator);

            badwordHashes = Word.words2hashesHandles(badwords);

            this.log.logConfig("loaded badwords from file " + badwordsFile.getName() +

                               ", " + badwords.size() + " entries, " +

                               ppRamString(badwordsFile.length()/1024));

        }

        // load stopwords

        if (stopwords == null || stopwords.isEmpty()) {

            final File stopwordsFile = new File(appPath, SwitchboardConstants.LIST_STOPWORDS_DEFAULT);

            stopwords = SetTools.loadList(stopwordsFile, NaturalOrder.naturalComparator);

            stopwordHashes = Word.words2hashesHandles(stopwords);

            this.log.logConfig("loaded stopwords from file " + stopwordsFile.getName() + ", " +

            stopwords.size() + " entries, " +

            ppRamString(stopwordsFile.length()/1024));

        }

        // load ranking from distribution

        final File rankingPath = new File(this.appPath, "ranking/YBR".replace('/', File.separatorChar));

        BlockRank.loadBlockRankTable(rankingPath, 16);

        // load distributed ranking

        // very large memory configurations allow to re-compute a ranking table

        /*

        final File hostIndexFile = new File(this.queuesRoot, "hostIndex.blob");

        if (MemoryControl.available() > 1024 * 1024 * 1024) new Thread() {

            public void run() {

                ReferenceContainerCache<HostReference> hostIndex; // this will get large, more than 0.5 million entries by now

                if (!hostIndexFile.exists()) {

                    hostIndex = BlockRank.collect(Switchboard.this.peers, Switchboard.this.webStructure, Integer.MAX_VALUE);

                    BlockRank.saveHostIndex(hostIndex, hostIndexFile);

                } else {

                    hostIndex = BlockRank.loadHostIndex(hostIndexFile);

                }

                // use an index segment to find hosts for given host hashes

                final String segmentName = getConfig(SwitchboardConstants.SEGMENT_PUBLIC, "default");

                final Segment segment = Switchboard.this.indexSegments.segment(segmentName);

                final MetadataRepository metadata = segment.urlMetadata();

                Map<String,HostStat> hostHashResolver;

                try {

                    hostHashResolver = metadata.domainHashResolver(metadata.domainSampleCollector());

                } catch (final IOException e) {

                    hostHashResolver = new HashMap<String, HostStat>();

                }

                // recursively compute a new ranking table

                Switchboard.this.log.logInfo("BLOCK RANK: computing new ranking tables...");

                BlockRank.ybrTables = BlockRank.evaluate(hostIndex, hostHashResolver, null, 0);

                hostIndex = null; // we don't need that here any more, so free the memory

                // use the web structure and the hostHash resolver to analyse the ranking table

                Switchboard.this.log.logInfo("BLOCK RANK: analysis of " + BlockRank.ybrTables.length + " tables...");

                BlockRank.analyse(Switchboard.this.webStructure, hostHashResolver);

                // store the new table

                Switchboard.this.log.logInfo("BLOCK RANK: storing fresh table...");

                BlockRank.storeBlockRankTable(rankingPath);

            }

        }.start();

        */

        // load the robots.txt db

        this.log.logConfig("Initializing robots.txt DB");

        this.robots = new RobotsTxt(this.tables);

        this.log.logConfig("Loaded robots.txt DB: " +  this.robots.size() + " entries");

        // start a cache manager

        this.log.logConfig("Starting HT Cache Manager");

        // create the cache directory

        this.htCachePath = getDataPath(SwitchboardConstants.HTCACHE_PATH, SwitchboardConstants.HTCACHE_PATH_DEFAULT);

        this.log.logInfo("HTCACHE Path = " + this.htCachePath.getAbsolutePath());

        final long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(SwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte

        Cache.init(this.htCachePath, this.peers.mySeed().hash, maxCacheSize);

        // create the surrogates directories

        this.surrogatesInPath = getDataPath(SwitchboardConstants.SURROGATES_IN_PATH, SwitchboardConstants.SURROGATES_IN_PATH_DEFAULT);

        this.log.logInfo("surrogates.in Path = " + this.surrogatesInPath.getAbsolutePath());

        this.surrogatesInPath.mkdirs();

        this.surrogatesOutPath = getDataPath(SwitchboardConstants.SURROGATES_OUT_PATH, SwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT);

        this.log.logInfo("surrogates.out Path = " + this.surrogatesOutPath.getAbsolutePath());