parallel_tronconneuse.hpp

Go to the documentation of this file.

/*********************************************************************/
// dar - disk archive - a backup/restoration program
// Copyright (C) 2002-2025 Denis Corbin
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// to contact the author, see the AUTHOR file
/*********************************************************************/
 
 
 
#ifndef PARALLEL_TRONCONNEUSE_HPP
#define PARALLEL_TRONCONNEUSE_HPP
 
#include "../my_config.h"
#include <string>
 
#include "infinint.hpp"
#include "archive_version.hpp"
#include "crypto_segment.hpp"
#include "heap.hpp"
#include "crypto_module.hpp"
#include "proto_tronco.hpp"
 
#include <libthreadar/libthreadar.hpp>
 
namespace libdar
{
 
 
        // those class are used by the parallel_tronconneuse class to wrap the different
        // type of threads. They are defined just after the parallel_tronconneuse definition
    class read_below;
    class write_below;
    class crypto_worker;
 
 
    enum class tronco_flags { normal = 0, stop = 1, eof = 2, die = 3, data_error = 4, exception_below = 5, exception_worker = 6, exception_error = 7 };
 
 
        //
        // the parallel_tronconneuse class that orchestrate all that
        //
        //
 
 
 
 
    class parallel_tronconneuse : public proto_tronco
    {
    public:
 
        parallel_tronconneuse(U_I workers,
                              U_32 block_size,
                              generic_file & encrypted_side,
                              const archive_version & reading_ver,
                              std::unique_ptr<crypto_module> & ptr);
 
        parallel_tronconneuse(const parallel_tronconneuse & ref) = delete;
 
        parallel_tronconneuse(parallel_tronconneuse && ref) = default;
 
        parallel_tronconneuse & operator = (const parallel_tronconneuse & ref) = delete;
 
        parallel_tronconneuse & operator = (parallel_tronconneuse && ref) noexcept = default;
 
        ~parallel_tronconneuse() noexcept;
 
        virtual bool skippable(skippability direction, const infinint & amount) override;
        virtual bool skip(const infinint & pos) override;
        virtual bool skip_to_eof() override;
        virtual bool skip_relative(S_I x) override;
        virtual bool truncatable(const infinint & pos) const override { return false; };
        virtual infinint get_position() const override { if(is_terminated()) throw SRC_BUG; return current_position; };
 
 
        virtual void write_end_of_file() override { if(is_terminated()) throw SRC_BUG; sync_write(); };
 
 
 
        virtual void set_initial_shift(const infinint & x) override;
 
        virtual void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) override;
 
        virtual U_32 get_clear_block_size() const override { return clear_block_size; };
 
    private:
 
            // inherited from generic_file
 
        virtual void inherited_read_ahead(const infinint & amount) override;
 
        virtual U_I inherited_read(char *a, U_I size) override;
 
 
        virtual void inherited_write(const char *a, U_I size) override;
 
 
        virtual void inherited_truncate(const infinint & pos) override { throw SRC_BUG; };
 
        virtual void inherited_sync_write() override;
 
 
        virtual void inherited_flush_read() override;
 
        virtual void inherited_terminate() override;
 
        const archive_version & get_reading_version() const { return reading_ver; };
 
            // internal data structure
        enum class thread_status { running, suspended, dead };
 
            // the fields
 
        U_I num_workers;               
        U_32 clear_block_size;         
        infinint current_position;     
        infinint initial_shift;        
        archive_version reading_ver;   
        std::unique_ptr<crypto_module> crypto;  
        infinint (*mycallback)(generic_file & below, const archive_version & reading_ver);
        generic_file* encrypted;
 
            // fields used to represent possible status of subthreads and communication channel (the pipe)
 
        U_I ignore_stop_acks;          
        thread_status t_status;        
 
 
            // the following stores data from the ratelier_gather to be provided for read() operation
            // the lus_data/lus_flags is what is extracted from the ratelier_gather, both constitute
            // the feedback channel from sub-threads to provide order acks and normal data
 
        std::deque<std::unique_ptr<crypto_segment> > lus_data;
        std::deque<signed int> lus_flags;
        bool lus_eof;
        bool check_bytes_to_skip; 
 
            // the following stores data going to ratelier_scatter for the write() operation
 
        std::unique_ptr<crypto_segment> tempo_write;
        infinint block_num;
 
            // the datastructures shared among threads
 
        std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > scatter;
        std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > gather;
        std::shared_ptr<libthreadar::barrier> waiter;
        std::shared_ptr<heap<crypto_segment> > tas;
 
            // the child threads
 
        std::deque<std::unique_ptr<crypto_worker> > travailleur;
        std::unique_ptr<read_below> crypto_reader;
        std::unique_ptr<write_below> crypto_writer;
 
 
 
 
        bool send_read_order(tronco_flags order, const infinint & for_offset = 0);
 
        void send_write_order(tronco_flags order);
 
        void go_read();
 
        void read_refill();
 
 
        tronco_flags purge_ratelier_from_next_order(infinint pos = 0);
 
 
        bool purge_unack_stop_order(const infinint & pos = 0);
 
 
        bool find_offset_in_lus_data(const infinint & pos);
 
        void run_threads();
 
        void stop_threads();
 
    void join_workers_only();
 
        void join_threads();
 
 
        static U_I get_ratelier_size(U_I num_worker) { return num_worker + num_worker/2; };
        static U_I get_heap_size(U_I num_worker);
    };
 
 
        //
        // read_below subthread used by parallel_tronconneuse
        // to dispatch chunk of encrypted data to the workers
        //
 
    class read_below: public libthreadar::thread
    {
    public:
        read_below(const std::shared_ptr<libthreadar::ratelier_scatter<crypto_segment> > & to_workers, 
                   const std::shared_ptr<libthreadar::barrier> & waiter, 
                   U_I num_workers,              
                   U_I clear_block_size,         
                   generic_file* encrypted_side, 
                   const std::shared_ptr<heap<crypto_segment> > xtas, 
                   infinint init_shift           
        );
 
        ~read_below() { if(ptr) tas->put(std::move(ptr)); cancel(); join(); };
 
        void set_callback_trailing_clear_data(trailing_clear_data_callback call_back) { trailing_clear_data = call_back; };
 
            // *** //
            // *** the method above should not be used anymore once the thread is running *** //
            // *** //
 
        void set_initial_shift(const infinint & x) { initial_shift = x; };
 
 
        void set_pos(const infinint & pos) { skip_to = pos; };
 
 
        void set_flag(tronco_flags val) { flag = val; };
 
 
    const infinint & get_clear_flow_start() const { return clear_flow_start; };
 
 
    const infinint & get_pos_in_flow() const { return pos_in_flow; };
 
    void read_ahead_up_to(const infinint & offset);
 
 
    protected:
    virtual void inherited_run() override;
 
    private:
    std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > workers; 
    std::shared_ptr<libthreadar::barrier> waiting;                            
    U_I num_w;                                                                
    U_I clear_buf_size;                                                       
    generic_file* encrypted;                                                  
    archive_version version;                                                  
    std::shared_ptr<heap<crypto_segment> > tas;                               
    infinint initial_shift;                                                   
    bool reof;                                                                
    trailing_clear_data_callback trailing_clear_data;                         
    std::unique_ptr<crypto_segment> ptr;                                      
    infinint index_num;                                                       
    bool read_ahead_set;                                                      
    infinint read_ahead_offset;                                               
    libthreadar::mutex ra_cntrl;                                              
 
 
        // initialized by inherited_run() / get_ready_for_new_offset()
 
    U_I encrypted_buf_size;                                                   
 
        // fields accessible by both the caller and the read_below thread
 
    infinint skip_to;          
    tronco_flags flag;         
    infinint clear_flow_start; 
    infinint pos_in_flow;      
 
    void work();
    infinint get_ready_for_new_offset();
    void send_flag_to_workers(tronco_flags theflag);
 
        // same function as the tronconneuse::position_clear2crypt
    void position_clear2crypt(const infinint & pos,
                  infinint & file_buf_start,
                  infinint & clear_buf_start,
                  infinint & pos_in_buf,
                  infinint & block_num);
 
        // propagate read_ahead request if necessary
    void check_read_ahead();
    };
 
 
    //
    // write_below subthread used by parallel_tronconneuse
    // to gather and write down encrypted data work from workers
    //
 
    class write_below: public libthreadar::thread
    {
    public:
    write_below(const std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > & from_workers,
            const std::shared_ptr<libthreadar::barrier> & waiter,
            U_I num_workers,
            generic_file* encrypted_side,
            const std::shared_ptr<heap<crypto_segment> > xtas);
 
    ~write_below() { cancel(); join(); };
 
    bool exception_pending() const { return error; };
    const infinint & get_error_block() const { return error_block; };
 
    protected:
    virtual void inherited_run() override;
 
    private:
    std::shared_ptr<libthreadar::ratelier_gather<crypto_segment> > workers;
    std::shared_ptr<libthreadar::barrier> waiting;
    U_I num_w;
    U_I cur_num_w;
    generic_file* encrypted; 
    std::shared_ptr<heap<crypto_segment> > tas;
    bool error;
    infinint error_block; // last crypto block before error
    std::deque<std::unique_ptr<crypto_segment> >ones;
    std::deque<signed int> flags;
 
    void work();
    };
 
 
    //
    // the crypto_worker threads performing ciphering/deciphering
    // of many data blocks in parallel
    //
 
 
    class crypto_worker: public libthreadar::thread
    {
    public:
    crypto_worker(std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & read_side,
            std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & write_side,
            std::shared_ptr<libthreadar::barrier> waiter,
            std::unique_ptr<crypto_module> && ptr,
            bool encrypt);
 
    virtual ~crypto_worker() { cancel(); join(); };
 
    protected:
    virtual void inherited_run() override;
 
    private:
    enum class status { fine, inform, sent };
 
    std::shared_ptr<libthreadar::ratelier_scatter <crypto_segment> > & reader;
    std::shared_ptr<libthreadar::ratelier_gather <crypto_segment> > & writer;
    std::shared_ptr<libthreadar::barrier> waiting;
    std::unique_ptr<crypto_module> crypto;
    bool do_encrypt; // if false do decrypt
    std::unique_ptr<crypto_segment> ptr;
    unsigned int slot;
    status abort;
 
    void work();
    };
 
 
 
} // end of namespace
 
#endif