Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions flatdata-cpp/include/flatdata/ArchiveBuilder.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class ArchiveBuilder
const flatdata::ResourceStorage& storage( ) const;
bool is_created( ) const;
void check_created( ) const;
bool clean_up_signature( flatdata::ResourceStorage& storage );

private:
std::shared_ptr< flatdata::ResourceStorage > m_storage;
Expand Down
10 changes: 10 additions & 0 deletions flatdata-cpp/include/flatdata/FileResourceStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class FileResourceStorage : public ResourceStorage
std::unique_ptr< ResourceStorage > directory( const char* key ) override;

bool exists( const char* key ) override;
bool remove( const char* key ) override;

protected:
std::shared_ptr< std::ostream > create_output_stream( const char* key ) override;
Expand Down Expand Up @@ -150,4 +151,13 @@ FileResourceStorage::exists( const char* key )
return !ec;
}

inline bool
FileResourceStorage::remove( const char* key )
{
boost::system::error_code ec;
std::string file_path = get_path( key );
boost::filesystem::remove( file_path.c_str( ), ec );
return !ec;
}

} // namespace flatdata
22 changes: 22 additions & 0 deletions flatdata-cpp/include/flatdata/MemoryResourceStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ class MemoryResourceStorage : public ResourceStorage
public:
static std::unique_ptr< MemoryResourceStorage > create( );
bool exists( const char* key ) override;
bool remove( const char* key ) override;

std::unique_ptr< ResourceStorage > create_directory( const char* key ) override;
std::unique_ptr< ResourceStorage > directory( const char* key ) override;

Expand Down Expand Up @@ -259,6 +261,26 @@ MemoryResourceStorage::exists( const char* key )
|| helpers::prefix_exists( m_storage->streams, path + "/" );
}

inline bool
MemoryResourceStorage::remove( const char* key )
{
const auto path = get_path( key );
std::lock_guard< std::mutex > lock( m_storage_mutex );

m_storage->streams.erase( path );
auto found_resource = m_storage->resources.find( path );
if ( found_resource != m_storage->resources.end( ) )
{
#ifdef DEBUG_DATA_ACCESS_STATISTICS
auto& string = found_resource->second;
DebugDataAccessStatistics::deregister_mapping( MemoryDescriptor(
reinterpret_cast< const unsigned char* >( string->c_str( ) ), string->size( ) ) );
#endif
m_storage->resources.erase( found_resource );
}
return true;
}

template < typename ResourceSerializer >
std::string
MemoryResourceStorage::dump_resources( bool dump_schemas, ResourceSerializer&& f ) const
Expand Down
5 changes: 5 additions & 0 deletions flatdata-cpp/include/flatdata/ResourceStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ class ResourceStorage : private boost::noncopyable
* @brief Check if resource exists
*/
virtual bool exists( const char* key ) = 0;
virtual bool remove( const char* key ) = 0;

bool remove_resource( const char* resource_name, const char* expected_schema );
bool remove_multivector_index_resource( const char* resource_name,
const char* expected_schema );

protected:
/**
Expand Down
8 changes: 8 additions & 0 deletions flatdata-cpp/include/flatdata/TarArchiveResourceStorage.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class TarArchiveResourceStorage : public ResourceStorage
std::unique_ptr< ResourceStorage > create_directory( const char* key ) override;
std::unique_ptr< ResourceStorage > directory( const char* key ) override;
bool exists( const char* key ) override;
bool remove( const char* key ) override;

protected:
std::shared_ptr< std::ostream > create_output_stream( const char* key ) override;
Expand Down Expand Up @@ -123,4 +124,11 @@ TarArchiveResourceStorage::exists( const char* key )
return m_storage->read( get_path( key ).c_str( ) ).data( ) != nullptr;
}

inline bool
TarArchiveResourceStorage::remove( const char* /* key */ )
{
// Removing parts of TAR files is not supported
return false;
}

} // namespace flatdata
38 changes: 38 additions & 0 deletions flatdata-cpp/include/flatdata/internal/ResourceStorage.inl
Original file line number Diff line number Diff line change
Expand Up @@ -207,4 +207,42 @@ ResourceStorage::create_multi_vector( const char* resource_name, const char* sch
internal::ResourceReader( this, resource_name, schema ) ) );
}

inline bool
ResourceStorage::remove_resource( const char* resource_name, const char* expected_schema )
{
const std::string schema_name = std::string( resource_name ) + ".schema";
Comment thread
ChaoJia marked this conversation as resolved.
Resolved
const bool resource_exists = exists( resource_name );
const bool schema_exists = exists( schema_name.c_str( ) );

if ( !resource_exists && !schema_exists )
{
return true;
}

if ( resource_exists != schema_exists )
{
return false;
}

auto schema = read_resource( ( schema_name.c_str( ) ) );
std::string stored_schema( reinterpret_cast< const char* >( schema.data( ) ),
schema.size_in_bytes( ) );

if ( stored_schema != expected_schema )
{
return false;
}
return remove( resource_name ) && remove( schema_name.c_str( ) );
}

inline bool
ResourceStorage::remove_multivector_index_resource( const char* resource_name,
const char* expected_schema )
{
std::string index_name = std::string( resource_name ) + internal::INDEX_SUFFIX;
std::string index_schema = internal::multivector_index_schema( expected_schema );

return remove_resource( index_name.c_str( ), index_schema.c_str( ) );
}

} // namespace flatdata
7 changes: 7 additions & 0 deletions flatdata-cpp/src/ArchiveBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,11 @@ ArchiveBuilder::initialize( )
return static_cast< bool >( signature );
}

bool
ArchiveBuilder::clean_up_signature( flatdata::ResourceStorage& storage )
{
const auto signature_name = internal::signature_name( name( ) );
return storage.remove_resource( signature_name.c_str( ), schema( ) );
}

} // namespace flatdata
23 changes: 23 additions & 0 deletions flatdata-cpp/test/ResourceStorageTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -81,5 +81,28 @@ TEST_CASE( "schema is checked for written vector", "[ResourceStorage]" )
( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) );
}

TEST_CASE( "resource doesn't exist after removing it", "[ResourceStorage]" )
{
std::unique_ptr< ResourceStorage > a = MemoryResourceStorage::create( );
auto create = [ & ] {
auto v = a->create_external_vector< AStruct >( resource_name, correct_schema );
v.close( );
};
REQUIRE_NOTHROW( create( ) );

REQUIRE_FALSE(
( a->read< ArrayView< AStruct > >( resource_name, incorrect_schema ).is_initialized( ) ) );
REQUIRE(
( a->read< ArrayView< AStruct > >( resource_name, correct_schema ).is_initialized( ) ) );

std::string schema_name = std::string( resource_name ) + ".schema";
REQUIRE( a->exists( resource_name ) );
REQUIRE( a->exists( schema_name.c_str( ) ) );

a->remove_resource( resource_name, correct_schema );
REQUIRE_FALSE( a->exists( resource_name ) );
REQUIRE_FALSE( a->exists( schema_name.c_str( ) ) );
}

} // namespace test
} // namespace flatdata
63 changes: 50 additions & 13 deletions flatdata-generator/flatdata/generator/templates/cpp/archive.jinja2
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class {{ builder_name }} : public flatdata::ArchiveBuilder
{
public:
/// Creates Archive builder
static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage );
static {{ builder_name }} open( std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite = false );
/// Archive schema
static const char* schema_definition( );

Expand All @@ -65,18 +65,18 @@ public: /// Resources
using {{ r|typedef_name }} = {{ resource.writer_type(r) }};
using {{ r|typedef_name("Reader") }} = {{ resource.reader_type(r) }};
{% if r|resource_provides_incremental_builder %}
{{ r|typedef_name }} start_{{ r.name }}( );
{{ r|typedef_name }} start_{{ r.name }}( bool force_overwrite = false );
{% endif %}
{% if r|provides_setter %}
bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data );
bool set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite = false );
{% endif %}

static bool clean_up_{{ r.name }}( flatdata::ResourceStorage& storage );
{% endfor %}

{% for r in archive.resources|archive_resources %}
using {{ r|typedef_name }} = {{ resource.writer_type(r) }};
{{ r|typedef_name }}& {{ r.name }}( );
{{ r|typedef_name }}& {{ r.name }}( bool force_overwrite = false );
{% endfor %}
bool clean_up( flatdata::ResourceStorage& storage );

private:
{{ builder_name }}( std::shared_ptr< flatdata::ResourceStorage > storage );
Expand Down Expand Up @@ -205,9 +205,13 @@ inline


inline {{ builder_name }}
{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage )
{{ builder_name }}::open(std::shared_ptr< flatdata::ResourceStorage > storage, bool force_overwrite )
{
{{ builder_name }} result( storage );
if ( force_overwrite && !result.clean_up( *storage ) )
{
return {{ builder_name }}( );
}
if ( !result.initialize( ) )
{
return {{ builder_name }}( );
Expand All @@ -217,31 +221,64 @@ inline {{ builder_name }}

{% for r in archive.resources|simple_resources %}
{% if r|resource_provides_incremental_builder %}
inline auto {{ builder_name }}::start_{{ r.name }}( ) -> {{ r|typedef_name }}
inline auto {{ builder_name }}::start_{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }}
{
check_created( );
if ( !force_overwrite || !clean_up_{{ r.name }}( storage( ) ) )
{
check_created( );
}
return storage( ).{{ resource.incremental_write_call(r) }}( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
}
{% endif %}
{% if r|provides_setter %}
inline bool
{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data )
{{ builder_name }}::set_{{ r.name }}( {{ r|typedef_name("Reader") }} data, bool force_overwrite )
{
check_created( );
if ( !force_overwrite || !clean_up_{{ r.name }}( storage( ) ) )
{
check_created( );
}
return storage( ).write< {{ r|typedef_name("Reader") }} >( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__, data );
}
{% endif %}
inline bool
{{ builder_name }}::clean_up_{{ r.name }}( flatdata::ResourceStorage& storage )
{
bool res = storage.remove_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
{% if r|is_multivector %}
res = res && storage.remove_multivector_index_resource( "{{ r.name }}", internal::{{ archive.name }}__{{ r.name }}__schema__ );
{% endif %}
return res;
}
{% endfor %}

{% for r in archive.resources|archive_resources %}
inline auto
{{ builder_name }}::{{ r.name }}( ) -> {{ r|typedef_name }}&
{{ builder_name }}::{{ r.name }}( bool force_overwrite ) -> {{ r|typedef_name }}&
{
if ( !m_{{ r.name }}.is_open( ) )
{
m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ) );
m_{{ r.name }} = {{ r|typedef_name }}::open( this->storage( ).create_directory( "{{ r.name }}" ), force_overwrite );
}
return m_{{ r.name }};
}
{% endfor %}

inline bool
{{ builder_name }}::clean_up( flatdata::ResourceStorage& storage )
{
bool res = true;
{% for r in archive.resources|simple_resources %}
res = res && {{ builder_name }}::clean_up_{{ r.name }}( storage );
{% endfor %}
{% for r in archive.resources|archive_resources %}
if ( auto subarchive_storage = storage.directory( "{{ r.name }}" ) )
{
res = res && m_{{ r.name }}.clean_up( *subarchive_storage );
}
{% endfor %}
res = res && clean_up_signature( storage );
return res;
}

{% endmacro %}
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,31 @@ impl {{archive.name}}Builder {
}

{% endif %}
#[inline]
pub fn clean_up_{{ r.name }}_storage(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> {
{% if [r] | subarchive_resources %}
{% set t = fully_qualified_name(archive, r.target.node) %}
{{t}}Builder::clean_up(storage.subdir("{{r.name}}"))
{% else %}
{% if [r] | multivector_resources %}
storage.remove_multivector_index_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }})?;
{% endif %}
storage.remove_resource("{{r.name}}", schema::{{ archive_ns }}::resources::{{ r.name | upper }})
{% endif %}
}
#[inline]
pub fn clean_up_{{ r.name }}(&self) -> Result<(), ::std::io::Error> {
Self::clean_up_{{ r.name }}_storage(self.storage.clone())
}
{% endfor %}

#[inline]
pub fn clean_up(storage: flatdata::StorageHandle) -> Result<(), ::std::io::Error> {
{% for r in archive.resources | supported_resources %}
Self::clean_up_{{ r.name }}_storage(storage.clone())?;
{% endfor %}
storage.remove_resource("{{archive.name}}.archive", schema::{{ archive_ns }}::{{ archive.name | camel_to_snake_case | upper }})
}
}
{% endif %}

Expand Down
10 changes: 10 additions & 0 deletions flatdata-rs/lib/src/filestorage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,16 @@ impl ResourceStorage for FileResourceStorage {
self.path.join(resource_name).exists()
}

fn remove(&self, resource_name: &str) -> Result<(), io::Error> {
let resource_path = self.path.join(resource_name);
fs::remove_file(&resource_path).map_err(|e| {
io::Error::new(
e.kind(),
String::from(resource_path.to_str().unwrap_or(resource_name)),
)
})
}

fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> {
let resource_path = self.path.join(resource_name);
if !resource_path.exists() {
Expand Down
11 changes: 11 additions & 0 deletions flatdata-rs/lib/src/memstorage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,17 @@ impl ResourceStorage for MemoryResourceStorage {
.contains_key(&resource_path)
}

fn remove(&self, resource_name: &str) -> Result<(), io::Error> {
let resource_path = self.path.join(resource_name);
self.storage.streams.lock().unwrap().remove(&resource_path);
self.storage
.resources
.lock()
.unwrap()
.remove(&resource_path);
Ok(())
}

fn read_resource(&self, resource_name: &str) -> Result<&[u8], io::Error> {
let resource_path = self.path.join(resource_name);
if !self
Expand Down
Loading
Loading