IBL 文件整理器

//
// BoostTest01.cpp : 定義主控台應用程式的進入點。
//
#include "stdafx.h"

#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ini_parser.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <boost/regex.hpp>
#include <boost/foreach.hpp>

#include <boost/logic/tribool.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/date_time/local_time/local_time.hpp>
#include <boost/date_time/local_time/local_time_io.hpp>
#include <typeinfo>

#include <iostream>
#include <codecvt>
#include <fstream>
#include <set>
#include <map>

#include <vector>
#include <algorithm>

typedef std::set<boost::filesystem::wpath>                  myFolderList;
typedef myFolderList::key_type                                myFolderList_Key;

typedef std::map<unsigned __int64, std::set<boost::filesystem::wpath>>    myContainer;
typedef myContainer::key_type                                myContainer_Key;
typedef myContainer::value_type                                myContainer_Value;
typedef myContainer::value_type::first_type                    myContainer_DocID;
typedef myContainer::value_type::second_type                myContainer_PathSet;
typedef myContainer::value_type::second_type::value_type    myContainer_Path;

typedef std::map<unsigned __int64, std::wstring>            myDocTitle;
typedef myDocTitle::key_type                                myDocTitle_Key;
typedef myDocTitle::value_type                                myDocTitle_Value;
typedef myDocTitle::value_type::first_type                    myDocTitle_DocID;
typedef myDocTitle::value_type::second_type                    myDocTitle_Title;

typedef boost::logic::tribool                                myBoolean;
typedef std::map<boost::filesystem::wpath, std::vector<boost::filesystem::wpath>>    myTestContainer;

static void scanFilesUseRecursive(const std::wstring &rootPath, myContainer &rt = myContainer()) throw() {
    myFolderList_Key fullpath(rootPath, boost::filesystem::native);

    if(!boost::filesystem::exists(fullpath)) {
        return;
    }

    try {
        BOOST_FOREACH(const auto & value, boost::filesystem::wrecursive_directory_iterator(fullpath)) {
            if(boost::filesystem::is_directory(value)) {
                //std::wcout << boost::wformat(L"%1% is dir") % (*iter) << std::endl;
            } else {
                boost::wregex rx(L"^(\\d{6}).*?$");
                boost::wsmatch what;
                std::wstring FilePath(value.path().filename().wstring());

                if(boost::regex_match(FilePath, what, rx)) {
                    rt[boost::lexical_cast<myContainer_Key> (what.str(1))].insert(value.path());
                }
            }
        }
    } catch(const std::exception &ex) {
        std::wcout << ex.what() << std::endl;
    }
    return;
}

static void PTreePrint(std::wostream &os, const boost::property_tree::wptree &pt, int tab) throw() {
    tab += 2;

    for each(const auto & value in pt) {
        os << std::wstring(tab, ' ');

        //os << boost::wformat(L"{%1%} [%2%]") % value.first % value.second.data() << std::endl;
        if(0 == value.second.data().size()) {
            os << boost::wformat(L"{%1%}") % value.first << std::endl;
        } else {
            os << boost::wformat(L"{%1%} [%2%]") % value.first % value.second.data() << std::endl;
        }

        PTreePrint(os, value.second, tab);
    }
}

static int ExtractInformation(std::wistream &ifs,
                              myFolderList &TargetPath = myFolderList(),
                              myFolderList &SourcePath = myFolderList(),
                              myFolderList_Key &DownloadLink = myFolderList_Key(),
                              myFolderList_Key &DownloadLinkUpdated = myFolderList_Key(),
                              myFolderList_Key &DownloadLinkNew = myFolderList_Key(),
                              myFolderList_Key &DebugOut = myFolderList_Key(),
                              myFolderList_Key &UpdateList = myFolderList_Key(),
                              myBoolean &KeepLastOne = myBoolean()) throw() {
    try {
        boost::property_tree::wptree pt;
        boost::property_tree::read_ini(ifs, pt);
        //PTreePrint (std::wcout, pt, 0);

        typedef boost::property_tree::wptree::path_type PTtype;

        for each(const auto & value in pt.get_child(PTtype(L"General_IBL.Source", L'/'))) {
            //std::wcout << boost::wformat(L"{%1%} [%2%]") % value.first % value.second.data() << std::endl;
            SourcePath.insert(value.second.data());
        }

        for each(const auto & value in pt.get_child(PTtype(L"General_IBL.Target", L'/'))) {
            //std::wcout << boost::wformat(L"{%1%} [%2%]") % value.first % value.second.data() << std::endl;
            TargetPath.insert(value.second.data());
        }

        UpdateList            = pt.get<myFolderList_Key>    (PTtype(L"General_IBL.Setting/UpdateList", L'/'));
        DownloadLink        = pt.get<myFolderList_Key>    (PTtype(L"General_IBL.Setting/DownloadLink", L'/'));
        DownloadLinkUpdated = pt.get<myFolderList_Key>    (PTtype(L"General_IBL.Setting/DownloadLinkUpdated", L'/'));
        DownloadLinkNew     = pt.get<myFolderList_Key>    (PTtype(L"General_IBL.Setting/DownloadLinkNew", L'/'));
        DebugOut            = pt.get<myFolderList_Key>    (PTtype(L"General_IBL.Setting/DebugOut", L'/'));
        KeepLastOne            = pt.get<bool>    (PTtype(L"General_IBL.Setting/KeepLastOne", L'/'));
    } catch(const std::exception &ex) {
        std::wcout << ex.what() << std::endl;
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}

static int ExtractDocTitle(std::wistream &ifs,
                           const myDocTitle_DocID &MaxDocID = myDocTitle_DocID(),
                           const myContainer &Target = myContainer(),
                           myDocTitle &DocUpdated = myDocTitle(),
                           myDocTitle &DocNew = myDocTitle()) throw() {
    try {
        boost::property_tree::wptree pt;
        boost::property_tree::read_json(ifs, pt);
        //PTreePrint (std::wcout, pt, 0);

        for each(const auto & value in pt.get_child(L"data")) {
            myDocTitle_DocID    DocId = value.second.get<unsigned __int64> (L"cntnt_meta_data_id");
            myDocTitle_Title    DocStr = value.second.get<std::wstring> (L"title_name");

            //if (MaxDocID < DocId) {
            //    DocNew[DocId] = DocStr;
            //} else if (Target.find (DocId) != Target.end()) {
            //    DocUpdated[DocId] = DocStr;
            //}

            DocNew[DocId] = DocStr;

            if(Target.find(DocId) != Target.end()) {
                DocUpdated[DocId] = DocStr;
            }
        }
    } catch(const std::exception &ex) {
        std::wcout << ex.what() << std::endl;
        return EXIT_FAILURE;
    }

    return EXIT_SUCCESS;
}

int _tmainbk8(int argc, _TCHAR *argv[]) throw() {
    //
    // Parsing argument count and vector
    //
    std::wcout.imbue(std::locale(""));
    std::wifstream ifs;

    switch(argc) {
    case 1:
        ifs.open("setting.ini", std::ios::binary);
        ifs.imbue(std::locale(ifs.getloc(), new std::codecvt_utf8_utf16<wchar_t>));
        break;

    default:
        return EXIT_FAILURE;
        break;
    }

    myFolderList TargetPaths, SourcePaths;
    myFolderList_Key DownloadLink, UpdateDownloadLink, DownloadLinkUpdate, DownloadLinkNew, DebugOut, UpdateList;
    myBoolean KeepLastOne = myBoolean::false_value;
    ExtractInformation(ifs, TargetPaths, SourcePaths, DownloadLink, DownloadLinkUpdate, DownloadLinkNew, DebugOut, UpdateList, KeepLastOne);
    ifs.close();

    std::wcout << L"Basic settings:" << std::endl;
    std::wcout << L"  General_IBL.Source.* = " << std::endl;

    for each(const auto & value in SourcePaths) {
        std::wcout << boost::wformat(L"    %1%") % value.wstring() << std::endl;
    }

    std::wcout << L"  General_IBL.Target.* = " << std::endl;

    for each(const auto & value in TargetPaths) {
        std::wcout << boost::wformat(L"    %1%") % value.wstring() << std::endl;
    }

    std::wcout << L"  General_IBL.Setting.* = " << std::endl;
    std::wcout << L"    .DownloadLink       = " << DownloadLink.wstring() << std::endl;
    std::wcout << L"    .DownloadLinkUpdate = " << DownloadLinkUpdate.wstring() << std::endl;
    std::wcout << L"    .DownloadLinkNew    = " << DownloadLinkNew.wstring() << std::endl;
    std::wcout << L"    .DebugOut           = " << DebugOut.wstring() << std::endl;
    std::wcout << L"    .UpdateList         = " << UpdateList.wstring() << std::endl;
    std::wcout << L"    .KeepLastOne        = " << std::boolalpha << KeepLastOne << std::noboolalpha << std::endl;
    std::wcout << std::endl;

    std::wofstream log(DebugOut.wstring());
    log << L'\xEF' << L'\xBB' << L'\xBF'; // UTF-8 BOM
    log.imbue(std::locale(log.getloc(), new std::codecvt_utf8_utf16<wchar_t>));

    //
    // Scan source folders and files to Map container (DocID to FullPath mapping)
    //
    myContainer mysource;

    for each(const auto & value in SourcePaths) {
        scanFilesUseRecursive(value.wstring(), mysource);
    }

    //
    // Debug output, Useless
    //
    std::wcout << boost::wformat(L"How many DocID were found in Source? %1%") % mysource.size() << std::endl;
    log << std::endl << boost::wformat(L"How many DocID were found in Source? %1%") % mysource.size() << std::endl;

    for each(const auto & value in mysource) {
        log << boost::wformat(L"[%1%]: S") % value.first << std::endl;

        for each(const auto & value2 in value.second) {
            log << boost::wformat(L"  %1%") % value2.wstring() << std::endl;
        }

        log << std::endl;
    }

    //
    // Scan target folders to Map container (DocID to folder mapping)
    //
    myContainer mytarget, mytargetOrg;

    for each(const auto & value in TargetPaths) {
        scanFilesUseRecursive(value.wstring(), mytarget);
    }

    std::copy(mytarget.begin(), mytarget.end(), std::inserter(mytargetOrg, mytargetOrg.end()));
    std::for_each(mytarget.begin(), mytarget.end(), [](myContainer_Value & value) throw() {
        //
        // Clear file name from mytarget container.
        //
        myContainer_PathSet TempSet;

        for each(const auto & value2 in value.second) {
            TempSet.insert(value2.parent_path());
        }

        value.second.swap(TempSet);
    });

    myContainer_Key MaxDocID = (0 < mytarget.size() ? mytarget.rbegin()->first : 0);
    std::wcout << boost::wformat(L"How many DocID were found in Target? %1%, Max DocID = %2%") % mytarget.size() % MaxDocID << std::endl;
    log << boost::wformat(L"How many DocID were found in Target? %1%, Max DocID = %2%") % mytarget.size() % MaxDocID << std::endl;

    //
    // Debug output, Useless
    //
    for each(const auto & value in mytarget) {
        log << boost::wformat(L"[%1%]: T") % value.first << std::endl;

        for each(const auto & value2 in value.second) {
            log << boost::wformat(L"  %1%") % value2.wstring() << std::endl;
        }

        if((mytargetOrg.find(value.first) != mytargetOrg.end()) && (value.second.size() != mytargetOrg.find(value.first)->second.size())) {
            log << boost::wformat(L"  Duplicate file list:") << std::endl;

            for each(const auto & value2 in mytargetOrg.find(value.first)->second) {
                log << boost::wformat(L"    %1%") % value2.wstring() << std::endl;
            }
        }

        log << std::endl;
    }

    std::wcout << std::endl;
    log << std::endl;

    //
    // Copy files from source to target. Delete it when success.
    //
    for each(const auto & value in mysource) {
        myContainer::iterator found = mytarget.find(value.first);

        if(found != mytarget.end()) {
            for each(const auto & value2 in value.second) {
                std::wcout << value2.filename().wstring() << std::endl;
                log << value2.filename().wstring() << std::endl;
                bool CopyStatus = false;

                //
                // copy it
                //
                for each(const auto & value3 in found->second) {
                    myFolderList_Key TempOutputPath(value3);
                    (TempOutputPath += "\\") += value2.filename();

                    try {
                        boost::filesystem::copy_file(value2, TempOutputPath, boost::filesystem::copy_option::overwrite_if_exists);
                        CopyStatus = (CopyStatus == false ? true : CopyStatus);
                        log << boost::wformat(L"Copy  : %1%\\ [ok]") % value3.wstring() << std::endl;
                    } catch(const std::exception &ex) {
                        std::wcout << boost::wformat(L"Copy  : %1%\\ [fail]") % value3.wstring() << std::endl;
                        std::wcout << ex.what() << std::endl;
                        log << boost::wformat(L"Copy  : %1%\\ [fail]") % value3.wstring() << std::endl;
                        log << ex.what() << std::endl;
                    }
                }

                //
                // delete it
                //
                if(CopyStatus == true) {
                    try {
                        boost::filesystem::remove(value2);
                        log << boost::wformat(L"Remove: %1% [ok]") % value2.wstring() << std::endl;
                    } catch(const std::exception &ex) {
                        std::wcout << boost::wformat(L"Remove: %1% [fail]") % value2.wstring() << std::endl;
                        std::wcout << ex.what() << std::endl;
                        log << boost::wformat(L"Remove: %1% [fail]") % value2.wstring() << std::endl;
                        log << ex.what() << std::endl;
                    }
                }

                log << std::endl;
            }
        } else {
            // if (found == mytarget.end() )
        }
    }

    if(myBoolean::true_value == KeepLastOne) {
        myContainer myDelList, MyDelList2;
        std::copy(mytargetOrg.begin(), mytargetOrg.end(), std::inserter(myDelList, myDelList.end()));

        for each(const auto & value in myDelList) {
            myTestContainer  test;

            for each(const auto & value2 in value.second) {
                if(true || (value2.extension() == L".pdf")) {
                    test[value2.parent_path()].push_back(value2);
                }
            }

            if(0 == test.size()) {
                continue;
            }

            myTestContainer  test2;

            for each(auto & value2 in test) {
                myTestContainer::value_type::second_type temp = value2.second;
                std::sort(temp.begin(), temp.end(), [](const boost::filesystem::wpath & a, const boost::filesystem::wpath & b) {
                    return boost::filesystem::last_write_time(b) < boost::filesystem::last_write_time(a);
                });

                temp.erase(temp.begin());

                if(0 != temp.size()) {
                    test2[value2.first] = temp;
                }
            }

            if(0 == test2.size()) {
                continue;
            }

            for each(const auto & value2 in test2) {
                for each(const auto & value3 in value2.second) {
                    MyDelList2[value.first].insert(value3);
                }
            }
        }

        log << std::endl;

        for each(const auto & value in MyDelList2) {
            log << boost::wformat(L"[%1%]: clear Duplicate file") % value.first << std::endl;

            for each(const auto & value2 in value.second) {
                //std::time_t t = boost::filesystem::last_write_time (value2);
                //log << boost::wformat (L"[%1%] %2%") % boost::posix_time::from_time_t (t) % value2 << std::endl;

                try {
                    boost::filesystem::remove(value2);
                    log << boost::wformat(L"Remove: %1% [ok]") % value2.wstring() << std::endl;
                } catch(const std::exception &ex) {
                    std::wcout << boost::wformat(L"Remove: %1% [fail]") % value2.wstring() << std::endl;
                    std::wcout << ex.what() << std::endl;
                    log << boost::wformat(L"Remove: %1% [fail]") % value2.wstring() << std::endl;
                    log << ex.what() << std::endl;
                }
            }
        }

    }

    log.close();

    std::wofstream ofs(DownloadLink.wstring());
    ofs << L'\xEF' << L'\xBB' << L'\xBF'; // UTF-8 BOM
    ofs.imbue(std::locale(ofs.getloc(), new std::codecvt_utf8_utf16<wchar_t>));

    for(const auto &value : mytarget) {
        //ofs << (boost::wformat (L"http://www.intel.com/cd/edesign/library/asmo-na/eng/%1%") % value.first) << std::endl;
        ofs << (boost::wformat(L"https://cdrd.intel.com/v1/dl/getContent/%1%") % value.first) << std::endl;
    }

    ofs.close();

    //std::wofstream ofs2 (DownloadLink.wstring());
    //ofs2 << L'\xEF' << L'\xBB' << L'\xBF'; // UTF-8 BOM
    //ofs2.imbue (std::locale (ofs2.getloc(), new std::codecvt_utf8_utf16<wchar_t>));
    //for (int i = 564337; i < 580000; ++i) {
    //    ofs2 << (boost::wformat (L"https://cdrd.intel.com/v1/dl/getContent/%1%") % i) << std::endl;
    //}
    //ofs2.close();

    // update url:
    // https://www-ssl.intel.com/libs/apps/intel/pod2/grdc.json/contentbydaterange

    ifs.open(UpdateList.wstring(), std::ios::binary);
    myDocTitle DocUpdated, DocNew;
    ExtractDocTitle(ifs, MaxDocID, mytarget, DocUpdated, DocNew);
    ifs.close();

    std::wcout << boost::wformat(L"How many DocID were found in DocUpdated? %1%") % DocUpdated.size() << std::endl;
    std::wcout << boost::wformat(L"How many DocID were found in DocNew?     %1%") % DocNew.size() << std::endl;

    std::wofstream ofs2(DownloadLinkUpdate.wstring());
    ofs2 << L'\xEF' << L'\xBB' << L'\xBF'; // UTF-8 BOM
    ofs2.imbue(std::locale(ofs.getloc(), new std::codecvt_utf8_utf16<wchar_t>));

    for each(const auto & value in DocUpdated) {
        // std::wcout << boost::wformat (L"[%1%] %2%") % value.first % value.second << std::endl;
        ofs2 << (boost::wformat(L"https://cdrd.intel.com/v1/dl/getContent/%1%") % value.first) << std::endl;
    }

    ofs2.close();

    std::wofstream ofs3(DownloadLinkNew.wstring());
    ofs3 << L'\xEF' << L'\xBB' << L'\xBF'; // UTF-8 BOM
    ofs3.imbue(std::locale(ofs.getloc(), new std::codecvt_utf8_utf16<wchar_t>));

    for each(const auto & value in DocNew) {
        ofs3 << boost::wformat(L"[%1%] %2%") % value.first % value.second << std::endl;
        ofs3 << (boost::wformat(L"https://cdrd.intel.com/v1/dl/getContent/%1%") % value.first) << std::endl;
        ofs3 << std::endl;
    }

    ofs3.close();

    return EXIT_SUCCESS;
}