/*
    libmaus2
    Copyright (C) 2009-2013 German Tischler
    Copyright (C) 2011-2013 Genome Research Limited

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#include <libmaus2/util/Utf8String.hpp>
#include <libmaus2/parallel/StdThread.hpp>

uint64_t libmaus2::util::Utf8StringBase::computeOctetLength(std::wistream & stream, uint64_t const len)
{
	::libmaus2::util::CountPutObject CPO;
	for ( uint64_t i = 0; i < len; ++i )
	{
		std::wistream::int_type const w = stream.get();
		assert ( w != std::wistream::traits_type::eof() );
		::libmaus2::util::UTF8::encodeUTF8(w,CPO);
	}

	return CPO.c;
}


void libmaus2::util::Utf8StringBase::setup()
{
	uint64_t const bitalign = 6*64;

	::libmaus2::rank::ImpCacheLineRank::unique_ptr_type tI(new ::libmaus2::rank::ImpCacheLineRank(((A.size()+(bitalign-1))/bitalign)*bitalign));
	I = std::move(tI);

	::libmaus2::rank::ImpCacheLineRank::WriteContext WC = I->getWriteContext();
	for ( uint64_t i = 0; i < A.size(); ++i )
		if ( (!(A[i] & 0x80)) || ((A[i] & 0xc0) != 0x80) )
			WC.writeBit(1);
		else
			WC.writeBit(0);

	for ( uint64_t i = A.size(); i % bitalign; ++i )
		WC.writeBit(0);

	WC.flush();

	::libmaus2::select::ImpCacheLineSelectSupport::unique_ptr_type tS(new ::libmaus2::select::ImpCacheLineSelectSupport(*I,8));
	S = std::move(tS);

	#if 0
	std::cerr << "A.size()=" << A.size() << std::endl;
	std::cerr << "I.byteSize()=" << I->byteSize() << std::endl;
	#endif
}

libmaus2::util::Utf8StringBase::Utf8StringBase(
	std::string const & filename,
	uint64_t offset,
	uint64_t blength,
	int const /* verbose */)
{
	::libmaus2::aio::InputStreamInstance CIS(filename);
	uint64_t const fs = ::libmaus2::util::GetFileSize::getFileSize(CIS);
	offset = std::min(offset,fs);
	blength = std::min(blength,fs-offset);

	CIS.seekg(offset);
	A = A_type(blength,false);
	CIS.read(reinterpret_cast<char *>(A.begin()),blength);

	setup();
}

libmaus2::util::Utf8StringBase::Utf8StringBase(std::istream & CIS, uint64_t blength, int const /* verbose */)
: A(blength,false)
{
	CIS.read(reinterpret_cast<char *>(A.begin()),blength);
	setup();
}

libmaus2::util::Utf8StringBase::Utf8StringBase(std::wistream & CIS, uint64_t const octetlength, uint64_t const symlength, int const /* verbose */)
: A(octetlength,false)
{
	::libmaus2::util::PutObject<uint8_t *> P(A.begin());
	for ( uint64_t i = 0; i < symlength; ++i )
	{
		std::wistream::int_type const w = CIS.get();
		assert ( w != std::wistream::traits_type::eof() );
		::libmaus2::util::UTF8::encodeUTF8(w,P);
	}

	setup();
}

::libmaus2::autoarray::AutoArray<uint64_t> libmaus2::util::Utf8StringBase::computePartStarts(
	::libmaus2::autoarray::AutoArray<uint8_t> const & A, uint64_t const tnumparts
)
{
	uint64_t const fs = A.size();
	uint64_t const tpartsize = (fs + tnumparts-1)/tnumparts;
	uint64_t const numparts = (fs + tpartsize-1)/tpartsize;
	::libmaus2::autoarray::AutoArray<uint64_t> partstarts(numparts+1,false);

	for ( int64_t i = 0; i < static_cast<int64_t>(numparts); ++i )
	{
		uint64_t j = std::min(i*tpartsize,fs);
		::libmaus2::util::GetCObject<uint8_t const *> G(A.begin()+j);

		while ( j != fs && ((G.get() & 0xc0) == 0x80) )
			++j;

		partstarts[i] = j;
	}

	partstarts[numparts] = fs;

	return partstarts;
}

::libmaus2::autoarray::AutoArray<uint64_t> libmaus2::util::Utf8StringBase::computePartStarts(
	std::string const & fn, uint64_t const tnumparts
)
{
	uint64_t const fs = ::libmaus2::util::GetFileSize::getFileSize(fn);
	uint64_t const tpartsize = (fs + tnumparts-1)/tnumparts;
	uint64_t const numparts = (fs + tpartsize-1)/tpartsize;
	::libmaus2::autoarray::AutoArray<uint64_t> partstarts(numparts+1,false);

	for ( int64_t i = 0; i < static_cast<int64_t>(numparts); ++i )
	{
		uint64_t j = std::min(i*tpartsize,fs);
		::libmaus2::aio::InputStreamInstance G(fn);
		G.seekg(j);

		while ( j != fs && ((G.get() & 0xc0) == 0x80) )
			++j;

		partstarts[i] = j;
	}

	partstarts[numparts] = fs;

	return partstarts;
}

template<typename _get_object_type>
struct HistogramThreadCallable : public ::libmaus2::parallel::StdThreadCallable
{
	typedef _get_object_type get_object_type;
	typedef HistogramThreadCallable<get_object_type> this_type;
	typedef std::unique_ptr<this_type> unique_ptr_type;

	get_object_type & G;
	uint64_t const tcodelen;
	::libmaus2::parallel::StdMutex & mutex;
	::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> & ESCH;
	uint64_t const t;

	HistogramThreadCallable(
		get_object_type & rG,
		uint64_t const rtcodelen,
		::libmaus2::parallel::StdMutex & rmutex,
		::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> & rESCH,
		uint64_t const rt
	)
	: G(rG), tcodelen(rtcodelen), mutex(rmutex), ESCH(rESCH), t(rt)
	{
		// startStack(32*1024);
	}

	void run()
	{
		try
		{
			::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> LESCH(8u);
			::libmaus2::autoarray::AutoArray<uint64_t> low(256);
			uint64_t codelen = 0;

			while ( codelen != tcodelen )
			{
				uint32_t const code = ::libmaus2::util::UTF8::decodeUTF8(G,codelen);

				if ( code < low.size() )
					low[code]++;
				else
				{
					if ( LESCH.loadFactor() > 0.8 )
						LESCH.extendInternal();
					LESCH.insert(code,1);
				}
			}

			// add low key counts to hash map
			for ( uint64_t i = 0; i < low.size(); ++i )
				if ( low[i] )
				{
					if ( LESCH.loadFactor() > 0.8 )
						LESCH.extendInternal();
					LESCH.insert(i,low[i]);
				}

			// add to global hash map
			{
				libmaus2::parallel::StdMutex::scope_lock_type smutex(mutex);
				for ( std::atomic < ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::key_type > const * ita =
					LESCH.begin(); ita != LESCH.end(); ++ita )
					if ( *ita != ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::unused() )
					{
						if ( ESCH.loadFactor() > 0.8 )
							ESCH.extendInternal();
						ESCH.insert(*ita,LESCH.getCount(*ita));
					}
			}
		}
		catch(std::exception const & ex)
		{
			std::cerr << ex.what() << std::endl;
		}
	}
};

::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > libmaus2::util::Utf8StringBase::getHistogramAsArray(::libmaus2::autoarray::AutoArray<uint8_t> const & A, uint64_t const numthreads)
{
	::libmaus2::autoarray::AutoArray<uint64_t> const partstarts = computePartStarts(A,numthreads);
	uint64_t const numparts = partstarts.size()-1;

	::libmaus2::parallel::StdMutex lock;
	::libmaus2::parallel::StdMutex mutex;
	::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> ESCH(8u);

	typedef HistogramThreadCallable< ::libmaus2::util::GetCObject<uint8_t const *> > callable_type;
	// typedef callable_type::unique_ptr_type callable_ptr_type;

	::libmaus2::autoarray::AutoArray< ::libmaus2::util::GetCObject<uint8_t const *>::unique_ptr_type > getters(numparts);
	// ::libmaus2::autoarray::AutoArray<callable_ptr_type> callables(numparts);
	::libmaus2::autoarray::AutoArray<libmaus2::parallel::StdThread::unique_ptr_type> threads(numparts);

	for ( uint64_t i = 0; i < numparts; ++i )
	{
		::libmaus2::util::GetCObject<uint8_t const *>::unique_ptr_type tgettersi(
                                new ::libmaus2::util::GetCObject<uint8_t const *>(A.begin()+partstarts[i])
                        );
		getters[i] = std::move(tgettersi);
		libmaus2::parallel::StdThreadCallable::unique_ptr_type tcalsi(new callable_type(*getters[i],
                                partstarts[i+1]-partstarts[i],mutex,ESCH,i));
		libmaus2::parallel::StdThread::unique_ptr_type tptr(new libmaus2::parallel::StdThread(tcalsi));
		threads[i] = std::move(tptr);
		threads[i]->start();
	}
	for ( uint64_t i = 0; i < numparts; ++i )
	{
		threads[i]->join();
		threads[i].reset();
	}

	::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > R(ESCH.size(),false);
	uint64_t p = 0;
	for ( std::atomic < ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::key_type > const * ita =
		ESCH.begin(); ita != ESCH.end(); ++ita )
		if ( *ita != ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::unused() )
			R [ p++ ] = std::pair<int64_t,uint64_t>(*ita,ESCH.getCount(*ita));

	std::sort(R.begin(),R.end());

	return R;
}

::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > libmaus2::util::Utf8StringBase::getHistogramAsArray(std::string const & fn, uint64_t const numthreads)
{
	::libmaus2::autoarray::AutoArray<uint64_t> const partstarts = computePartStarts(fn,numthreads);
	uint64_t const numparts = partstarts.size()-1;

	::libmaus2::parallel::StdMutex lock;
	::libmaus2::parallel::StdMutex mutex;
	::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t> ESCH(8u);

	typedef HistogramThreadCallable< ::libmaus2::aio::InputStreamInstance > call_type;
	typedef libmaus2::parallel::StdThread thread_type;
	typedef thread_type::unique_ptr_type thread_ptr_type;
	::libmaus2::autoarray::AutoArray< ::libmaus2::aio::InputStreamInstance::unique_ptr_type > getters(numparts);
	::libmaus2::autoarray::AutoArray<thread_ptr_type> threads(numparts);

	for ( uint64_t i = 0; i < numparts; ++i )
	{
		::libmaus2::aio::InputStreamInstance::unique_ptr_type tgettersi(
                                new ::libmaus2::aio::InputStreamInstance(fn)
                        );
		getters[i] = std::move(tgettersi);
		// getters[i]->setBufferSize(16*1024);
		getters[i]->seekg(partstarts[i]);

		libmaus2::parallel::StdThreadCallable::unique_ptr_type calptr(
			new call_type(*getters[i],
                                partstarts[i+1]-partstarts[i],mutex,ESCH,i)
		);

		thread_ptr_type tthreadsi(new thread_type(calptr));
		threads[i] = std::move(tthreadsi);
		threads[i]->start();
	}
	for ( uint64_t i = 0; i < numparts; ++i )
	{
		threads[i]->join();
		threads[i].reset();
	}

	::libmaus2::autoarray::AutoArray< std::pair<int64_t,uint64_t> > R(ESCH.size(),false);
	uint64_t p = 0;
	for ( std::atomic<::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::key_type> const * ita =
		ESCH.begin(); ita != ESCH.end(); ++ita )
		if ( *ita != ::libmaus2::util::ExtendingSimpleCountingHash<uint64_t,uint64_t>::unused() )
			R [ p++ ] = std::pair<int64_t,uint64_t>(*ita,ESCH.getCount(*ita));

	std::sort(R.begin(),R.end());

	return R;
}

::libmaus2::util::Histogram::unique_ptr_type libmaus2::util::Utf8StringBase::getHistogram(::libmaus2::autoarray::AutoArray<uint8_t> const & A, uint64_t const numthreads)
{
	::libmaus2::autoarray::AutoArray<uint64_t> const partstarts = computePartStarts(A,numthreads);
	uint64_t const numparts = partstarts.size()-1;

	::libmaus2::util::Histogram::unique_ptr_type hist(new ::libmaus2::util::Histogram);
	::libmaus2::parallel::StdMutex lock;

	#if defined(_OPENMP)
	#pragma omp parallel for num_threads(numthreads)
	#endif
	for ( int64_t t = 0; t < static_cast<int64_t>(numparts); ++t )
	{
		::libmaus2::util::Histogram::unique_ptr_type lhist(new ::libmaus2::util::Histogram);

		uint64_t codelen = 0;
		uint64_t const tcodelen = partstarts[t+1]-partstarts[t];
		::libmaus2::util::GetCObject<uint8_t const *> G(A.begin()+partstarts[t]);

		while ( codelen != tcodelen )
			(*lhist)(::libmaus2::util::UTF8::decodeUTF8(G,codelen));

		libmaus2::parallel::StdMutex::scope_lock_type slock(lock);
		hist->merge(*lhist);
	}

	return hist;
}

::libmaus2::util::Histogram::unique_ptr_type libmaus2::util::Utf8StringBase::getHistogram() const
{
	::libmaus2::util::Histogram::unique_ptr_type hist(new ::libmaus2::util::Histogram);

	for ( uint64_t i = 0; i < A.size(); ++i )
		if ( (A[i] & 0xc0) != 0x80 )
		{
			::libmaus2::util::GetCObject<uint8_t const *> G(A.begin()+i);
			wchar_t const v = ::libmaus2::util::UTF8::decodeUTF8(G);
			(*hist)(v);
		}

	return hist;
}

std::map<int64_t,uint64_t> libmaus2::util::Utf8StringBase::getHistogramAsMap() const
{
	::libmaus2::util::Histogram::unique_ptr_type hist(getHistogram());
	return hist->getByType<int64_t>();
}

std::map<int64_t,uint64_t> libmaus2::util::Utf8StringBase::getHistogramAsMap(::libmaus2::autoarray::AutoArray<uint8_t> const & A, uint64_t const numthreads)
{
	::libmaus2::util::Histogram::unique_ptr_type hist(getHistogram(A,numthreads));
	return hist->getByType<int64_t>();
}

template<unsigned int wordsize>
typename libmaus2::util::Utf8String<wordsize>::shared_ptr_type libmaus2::util::Utf8String<wordsize>::constructRaw(
	std::string const & filename,
	uint64_t const offset,
	uint64_t const blength
)
{
	return shared_ptr_type(new this_type(filename, offset, blength));
}


template<unsigned int wordsize>
::libmaus2::autoarray::AutoArray<typename libmaus2::util::Utf8String<wordsize>::saidx_t,static_cast<libmaus2::autoarray::alloc_type>(libmaus2::util::StringAllocTypes::sa_atype)>
	libmaus2::util::Utf8String<wordsize>::computeSuffixArray(bool const parallel) const
{
	if ( A.size() > static_cast<uint64_t>(::std::numeric_limits<saidx_t>::max()) )
	{
		::libmaus2::exception::LibMausException se;
		se.getStream() << "computeSuffixArray: input is too large for data type." << std::endl;
		se.finish();
		throw se;
	}

	::libmaus2::autoarray::AutoArray<saidx_t,static_cast<libmaus2::autoarray::alloc_type>(libmaus2::util::StringAllocTypes::sa_atype)> SA(A.size());
	if ( parallel )
		sort_type_parallel::divsufsort ( A.begin() , SA.begin() , A.size() );
	else
		sort_type_serial::divsufsort ( A.begin() , SA.begin() , A.size() );

	uint64_t p = 0;
	for ( uint64_t i = 0; i < SA.size(); ++i )
		if ( (A[SA[i]] & 0xc0) != 0x80 )
			SA[p++] = SA[i];
	SA.resize(p);

	for ( uint64_t i = 0; i < SA.size(); ++i )
	{
		assert ( (*I)[SA[i]] );
		SA[i] = I->rank1(SA[i])-1;
	}

	return SA;
}

template struct libmaus2::util::Utf8String<32>;
template struct libmaus2::util::Utf8String<64>;
